diff --git a/.appveyor.yml b/.appveyor.yml index 559431e2f9421f4cf1bfe64f1e2e9ee66b7bead5..fe4816688b43276a4a9ca7b911b39a43b8fc2141 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -3,6 +3,7 @@ os: Visual Studio 2015 environment: matrix: - ARCH: amd64 + - ARCH: x86 clone_folder: c:\dev\TDengine clone_depth: 1 @@ -23,6 +24,7 @@ notifications: - provider: Email to: - sangshuduo@gmail.com + on_build_success: true on_build_failure: true on_build_status_changed: true diff --git a/.travis.yml b/.travis.yml index eb69370418a9c83c7b8bfe5daa1d6ead19150243..f6a3900f7a87c1af58b757590ecc07f3bfe9ce53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -146,7 +146,7 @@ matrix: branch_pattern: coverity_scan - os: linux - dist: trusty + dist: xenial language: c git: - depth: 1 @@ -157,7 +157,7 @@ matrix: - build-essential - cmake env: - - DESC="trusty/gcc-4.8 build" + - DESC="xenial build" before_script: - export TZ=Asia/Harbin @@ -227,7 +227,7 @@ matrix: - os: linux arch: arm64 - dist: trusty + dist: xenial language: c git: - depth: 1 @@ -238,7 +238,7 @@ matrix: - build-essential - cmake env: - - DESC="trusty/gcc-4.8 build" + - DESC="xenial build" before_script: - export TZ=Asia/Harbin diff --git a/Jenkinsfile b/Jenkinsfile index edbe11d428d0381a81f64bbc823b5b51a5edf199..9544343bec6ef964fb15cf94c7a1a7c93d98810f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,92 +1,126 @@ +import hudson.model.Result +import jenkins.model.CauseOfInterruption +properties([pipelineTriggers([githubPush()])]) +node { + git url: 'https://github.com/taosdata/TDengine.git' +} + + +def abortPreviousBuilds() { + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() + + for (build in builds) { + if (!build.isBuilding()) { + continue; + } + + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } + + build.doKill() //doTerm(),doKill(),doTerm() + } +} +//abort previous build +abortPreviousBuilds() +def abort_previous(){ + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) +} +def pre_test(){ + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + sudo rmtaos + ''' + } + sh ''' + + cd ${WKC} + git checkout develop + git reset --hard HEAD~10 + git pull + git fetch + git checkout ${CHANGE_BRANCH} + git reset --hard HEAD~10 + git pull + git merge develop + cd ${WK} + git reset --hard HEAD~10 + git checkout develop + git pull + cd ${WK} + export TZ=Asia/Harbin + date + rm -rf ${WK}/debug + mkdir debug + cd debug + cmake .. > /dev/null + make > /dev/null + make install > /dev/null + cd ${WKC}/tests + ''' + return 1 +} pipeline { agent none + environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDinternal/community' } - + stages { + + stage('Parallel test stage') { + //only build pr + when { + changeRequest() + } parallel { - stage('pytest') { - agent{label '184'} + stage('python_1') { + agent{label 'p1'} steps { + + pre_test() sh ''' - date - cd ${WKC} - git reset --hard - git checkout develop - git pull - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - date - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null - make install > /dev/null cd ${WKC}/tests - #./test-all.sh smoke - ./test-all.sh pytest + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh p1 date''' } } - stage('test_b1') { - agent{label 'master'} + stage('python_2') { + agent{label 'p2'} steps { + + pre_test() + sh ''' + cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh p2 + date''' + } + } + stage('test_b1') { + agent{label 'b1'} + steps { + pre_test() sh ''' - cd ${WKC} - git reset --hard - git checkout develop - git pull - - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - date - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null cd ${WKC}/tests - #./test-all.sh smoke - ./test-all.sh b1 + ./test-all.sh b1fq date''' } } stage('test_crash_gen') { - agent{label "185"} + agent{label "b2"} steps { - sh ''' - cd ${WKC} - git reset --hard - git checkout develop - git pull - - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null - cd ${WKC}/tests/pytest - ''' + pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest @@ -102,200 +136,35 @@ pipeline { sh ''' date cd ${WKC}/tests - ./test-all.sh b2 + ./test-all.sh b2fq date ''' } } stage('test_valgrind') { - agent{label "186"} + agent{label "b3"} steps { + pre_test() + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + ./valgrind-test.sh 2>&1 > mem-error-out.log + ./handle_val_log.sh + ''' + } sh ''' - cd ${WKC} - git reset --hard - git checkout develop - git pull - - git submodule update - cd ${WK} - git reset --hard - git checkout develop - git pull - export TZ=Asia/Harbin - date - rm -rf ${WK}/debug - mkdir debug - cd debug - cmake .. > /dev/null - make > /dev/null - cd ${WKC}/tests/pytest - ./valgrind-test.sh 2>&1 > mem-error-out.log - ./handle_val_log.sh - date cd ${WKC}/tests - ./test-all.sh b3 + ./test-all.sh b3fq date''' } } - stage('connector'){ - agent{label "release"} - steps{ - sh''' - cd ${WORKSPACE} - git checkout develop - ''' - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/gotest - bash batchtest.sh - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker - python3 PythonChecker.py - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single >/dev/null - java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 - ''' - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# - dotnet run - ''' - } - - } - } - stage('arm64_build'){ - agent{label 'arm64'} - steps{ - sh ''' - cd ${WK} - git fetch - git checkout develop - git pull - cd ${WKC} - git fetch - git checkout develop - git pull - git submodule update - cd ${WKC}/packaging - ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0 - - ''' - } - } - stage('arm32_build'){ - agent{label 'arm32'} - steps{ - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WK} - git fetch - git checkout develop - git pull - cd ${WKC} - git fetch - git checkout develop - git pull - git submodule update - cd ${WKC}/packaging - ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0 - - ''' - } - - } - } - } + + } - } - post { - success { - emailext ( - subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' - - - - - - - - - - - - -

- 构建信息 -
-
    -
    -
  • 构建名称>>分支：${PROJECT_NAME}
    • -
  • 构建结果： Successful
    • -
  • 构建编号：${BUILD_NUMBER}
    • -
  • 触发用户：${CAUSE}
    • -
  • 变更概要：${CHANGES}
    • -
  • 构建地址：${BUILD_URL}
    • -
  • 构建日志：${BUILD_URL}console
    • -
  • 变更集：${JELLY_SCRIPT}
    • -
    -
-
- - ''', - to: "yqliu@taosdata.com,pxiao@taosdata.com", - from: "support@taosdata.com" - ) - } - failure { - emailext ( - subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: ''' - - - - - - - - - - - - -

- 构建信息 -
-
    -
    -
  • 构建名称>>分支：${PROJECT_NAME}
    • -
  • 构建结果： Successful
    • -
  • 构建编号：${BUILD_NUMBER}
    • -
  • 触发用户：${CAUSE}
    • -
  • 变更概要：${CHANGES}
    • -
  • 构建地址：${BUILD_URL}
    • -
  • 构建日志：${BUILD_URL}console
    • -
  • 变更集：${JELLY_SCRIPT}
    • -
    -
-
- - ''', - to: "yqliu@taosdata.com,pxiao@taosdata.com", - from: "support@taosdata.com" - ) - } - } -} \ No newline at end of file + } + +} diff --git a/alert/go.mod b/alert/go.mod index 01c557d5646714869ad263a9c8fe2fa262fe9a43..43920f1f1aea2a95d02fbaddeab8ebc80ad1e81f 100644 --- a/alert/go.mod +++ b/alert/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/jmoiron/sqlx v1.2.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible - github.com/taosdata/driver-go v0.0.0-20200727182616-1a3b1941c206 + github.com/taosdata/driver-go v0.0.0-20201113094317-050667e5b4d0 go.uber.org/zap v1.14.1 google.golang.org/appengine v1.6.5 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c diff --git a/cmake/define.inc b/cmake/define.inc index 782dc625bf5f82b09637648063649b944a7747e3..5d4d94ff42c73a7ca11f32f8318526bfcfe4e525 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -45,7 +45,7 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) MESSAGE(STATUS "linux64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () @@ -53,7 +53,7 @@ IF (TD_LINUX_32) ADD_DEFINITIONS(-D_TD_LINUX_32) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "linux32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_64) @@ -62,7 +62,7 @@ IF (TD_ARM_64) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_ARM_32) @@ -70,21 +70,21 @@ IF (TD_ARM_32) ADD_DEFINITIONS(-D_TD_ARM_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "arm32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ") ENDIF () IF (TD_MIPS_64) ADD_DEFINITIONS(-D_TD_MIPS_64_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_MIPS_32) ADD_DEFINITIONS(-D_TD_MIPS_32_) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "mips32 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () IF (TD_APLHINE) @@ -105,8 +105,8 @@ IF (TD_LINUX) MESSAGE(STATUS "set ningsi macro to true") ENDIF () - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0 -Wno-unused-variable -Wunused-but-set-variable") + SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") + SET(RELEASE_FLAGS "-O3 -Wno-error") IF (${COVER} MATCHES "true") MESSAGE(STATUS "Test coverage mode, add extra flags") @@ -125,9 +125,9 @@ IF (TD_DARWIN_64) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) ADD_DEFINITIONS(-DUSE_LIBICONV) MESSAGE(STATUS "darwin64 is defined") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") + SET(RELEASE_FLAGS "-Og") ENDIF () IF (TD_WINDOWS) @@ -140,7 +140,7 @@ IF (TD_WINDOWS) IF (NOT TD_GODLL) SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") SET(DEBUG_FLAGS "/Zi /W3 /GL") - SET(RELEASE_FLAGS "/W0 /GL") + SET(RELEASE_FLAGS "/W0 /O3 /GL") ENDIF () INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) diff --git a/cmake/env.inc b/cmake/env.inc index 18a6fea51d7ab4c6a4068efafc478360737df9be..efcc9961767aec8555897e7a96fec91356017659 100755 --- a/cmake/env.inc +++ b/cmake/env.inc @@ -41,8 +41,10 @@ SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FL # SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}") IF (${CMAKE_BUILD_TYPE} MATCHES "Debug") + SET(CMAKE_BUILD_TYPE "Debug") MESSAGE(STATUS "Build Debug Version") ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release") + SET(CMAKE_BUILD_TYPE "Release") MESSAGE(STATUS "Build Release Version") ELSE () IF (TD_WINDOWS) diff --git a/cmake/version.inc b/cmake/version.inc index 948c7d2d0b7181cc29d91884166c844f39d8bf6b..7d0ad0585f67921c8c2ddbcf47ba08f803cc375a 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.9.0") + SET(TD_VER_NUMBER "2.0.12.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/deps/libcurl/include/curl/curl.h b/deps/libcurl/include/curl/curl.h new file mode 100644 index 0000000000000000000000000000000000000000..84229bb698b77a5ed5d67057772ed23188e191e8 --- /dev/null +++ b/deps/libcurl/include/curl/curl.h @@ -0,0 +1,2415 @@ +#ifndef __CURL_CURL_H +#define __CURL_CURL_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* + * If you have libcurl problems, all docs and details are found here: + * http://curl.haxx.se/libcurl/ + * + * curl-library mailing list subscription and unsubscription web interface: + * http://cool.haxx.se/mailman/listinfo/curl-library/ + */ + +#include "curlver.h" /* libcurl version defines */ +#include "curlbuild.h" /* libcurl build definitions */ +#include "curlrules.h" /* libcurl rules enforcement */ + +/* + * Define WIN32 when build target is Win32 API + */ + +#if (defined(_WIN32) || defined(__WIN32__)) && \ + !defined(WIN32) && !defined(__SYMBIAN32__) +#define WIN32 +#endif + +#include +#include + +#if defined(__FreeBSD__) && (__FreeBSD__ >= 2) +/* Needed for __FreeBSD_version symbol definition */ +#include +#endif + +/* The include stuff here below is mainly for time_t! */ +#include +#include + +#if defined(WIN32) && !defined(_WIN32_WCE) && !defined(__CYGWIN__) +#if !(defined(_WINSOCKAPI_) || defined(_WINSOCK_H) || \ + defined(__LWIP_OPT_H__) || defined(LWIP_HDR_OPT_H)) +/* The check above prevents the winsock2 inclusion if winsock.h already was + included, since they can't co-exist without problems */ +#include +#include +#endif +#endif + +/* HP-UX systems version 9, 10 and 11 lack sys/select.h and so does oldish + libc5-based Linux systems. Only include it on systems that are known to + require it! */ +#if defined(_AIX) || defined(__NOVELL_LIBC__) || defined(__NetBSD__) || \ + defined(__minix) || defined(__SYMBIAN32__) || defined(__INTEGRITY) || \ + defined(ANDROID) || defined(__ANDROID__) || defined(__OpenBSD__) || \ + (defined(__FreeBSD_version) && (__FreeBSD_version < 800000)) +#include +#endif + +#if !defined(WIN32) && !defined(_WIN32_WCE) +#include +#endif + +#if !defined(WIN32) && !defined(__WATCOMC__) && !defined(__VXWORKS__) +#include +#endif + +#ifdef __BEOS__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void CURL; + +/* + * libcurl external API function linkage decorations. + */ + +#ifdef CURL_STATICLIB +# define CURL_EXTERN +#elif defined(WIN32) || defined(_WIN32) || defined(__SYMBIAN32__) +# if defined(BUILDING_LIBCURL) +# define CURL_EXTERN __declspec(dllexport) +# else +# define CURL_EXTERN __declspec(dllimport) +# endif +#elif defined(BUILDING_LIBCURL) && defined(CURL_HIDDEN_SYMBOLS) +# define CURL_EXTERN CURL_EXTERN_SYMBOL +#else +# define CURL_EXTERN +#endif + +#ifndef curl_socket_typedef +/* socket typedef */ +#if defined(WIN32) && !defined(__LWIP_OPT_H__) && !defined(LWIP_HDR_OPT_H) +typedef SOCKET curl_socket_t; +#define CURL_SOCKET_BAD INVALID_SOCKET +#else +typedef int curl_socket_t; +#define CURL_SOCKET_BAD -1 +#endif +#define curl_socket_typedef +#endif /* curl_socket_typedef */ + +struct curl_httppost { + struct curl_httppost *next; /* next entry in the list */ + char *name; /* pointer to allocated name */ + long namelength; /* length of name length */ + char *contents; /* pointer to allocated data contents */ + long contentslength; /* length of contents field, see also + CURL_HTTPPOST_LARGE */ + char *buffer; /* pointer to allocated buffer contents */ + long bufferlength; /* length of buffer field */ + char *contenttype; /* Content-Type */ + struct curl_slist* contentheader; /* list of extra headers for this form */ + struct curl_httppost *more; /* if one field name has more than one + file, this link should link to following + files */ + long flags; /* as defined below */ + +/* specified content is a file name */ +#define CURL_HTTPPOST_FILENAME (1<<0) +/* specified content is a file name */ +#define CURL_HTTPPOST_READFILE (1<<1) +/* name is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRNAME (1<<2) +/* contents is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRCONTENTS (1<<3) +/* upload file from buffer */ +#define CURL_HTTPPOST_BUFFER (1<<4) +/* upload file from pointer contents */ +#define CURL_HTTPPOST_PTRBUFFER (1<<5) +/* upload file contents by using the regular read callback to get the data and + pass the given pointer as custom pointer */ +#define CURL_HTTPPOST_CALLBACK (1<<6) +/* use size in 'contentlen', added in 7.46.0 */ +#define CURL_HTTPPOST_LARGE (1<<7) + + char *showfilename; /* The file name to show. If not set, the + actual file name will be used (if this + is a file part) */ + void *userp; /* custom pointer used for + HTTPPOST_CALLBACK posts */ + curl_off_t contentlen; /* alternative length of contents + field. Used if CURL_HTTPPOST_LARGE is + set. Added in 7.46.0 */ +}; + +/* This is the CURLOPT_PROGRESSFUNCTION callback proto. It is now considered + deprecated but was the only choice up until 7.31.0 */ +typedef int (*curl_progress_callback)(void *clientp, + double dltotal, + double dlnow, + double ultotal, + double ulnow); + +/* This is the CURLOPT_XFERINFOFUNCTION callback proto. It was introduced in + 7.32.0, it avoids floating point and provides more detailed information. */ +typedef int (*curl_xferinfo_callback)(void *clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow); + +#ifndef CURL_MAX_WRITE_SIZE + /* Tests have proven that 20K is a very bad buffer size for uploads on + Windows, while 16K for some odd reason performed a lot better. + We do the ifndef check to allow this value to easier be changed at build + time for those who feel adventurous. The practical minimum is about + 400 bytes since libcurl uses a buffer of this size as a scratch area + (unrelated to network send operations). */ +#define CURL_MAX_WRITE_SIZE 16384 +#endif + +#ifndef CURL_MAX_HTTP_HEADER +/* The only reason to have a max limit for this is to avoid the risk of a bad + server feeding libcurl with a never-ending header that will cause reallocs + infinitely */ +#define CURL_MAX_HTTP_HEADER (100*1024) +#endif + +/* This is a magic return code for the write callback that, when returned, + will signal libcurl to pause receiving on the current transfer. */ +#define CURL_WRITEFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_write_callback)(char *buffer, + size_t size, + size_t nitems, + void *outstream); + + + +/* enumeration of file types */ +typedef enum { + CURLFILETYPE_FILE = 0, + CURLFILETYPE_DIRECTORY, + CURLFILETYPE_SYMLINK, + CURLFILETYPE_DEVICE_BLOCK, + CURLFILETYPE_DEVICE_CHAR, + CURLFILETYPE_NAMEDPIPE, + CURLFILETYPE_SOCKET, + CURLFILETYPE_DOOR, /* is possible only on Sun Solaris now */ + + CURLFILETYPE_UNKNOWN /* should never occur */ +} curlfiletype; + +#define CURLFINFOFLAG_KNOWN_FILENAME (1<<0) +#define CURLFINFOFLAG_KNOWN_FILETYPE (1<<1) +#define CURLFINFOFLAG_KNOWN_TIME (1<<2) +#define CURLFINFOFLAG_KNOWN_PERM (1<<3) +#define CURLFINFOFLAG_KNOWN_UID (1<<4) +#define CURLFINFOFLAG_KNOWN_GID (1<<5) +#define CURLFINFOFLAG_KNOWN_SIZE (1<<6) +#define CURLFINFOFLAG_KNOWN_HLINKCOUNT (1<<7) + +/* Content of this structure depends on information which is known and is + achievable (e.g. by FTP LIST parsing). Please see the url_easy_setopt(3) man + page for callbacks returning this structure -- some fields are mandatory, + some others are optional. The FLAG field has special meaning. */ +struct curl_fileinfo { + char *filename; + curlfiletype filetype; + time_t time; + unsigned int perm; + int uid; + int gid; + curl_off_t size; + long int hardlinks; + + struct { + /* If some of these fields is not NULL, it is a pointer to b_data. */ + char *time; + char *perm; + char *user; + char *group; + char *target; /* pointer to the target filename of a symlink */ + } strings; + + unsigned int flags; + + /* used internally */ + char * b_data; + size_t b_size; + size_t b_used; +}; + +/* return codes for CURLOPT_CHUNK_BGN_FUNCTION */ +#define CURL_CHUNK_BGN_FUNC_OK 0 +#define CURL_CHUNK_BGN_FUNC_FAIL 1 /* tell the lib to end the task */ +#define CURL_CHUNK_BGN_FUNC_SKIP 2 /* skip this chunk over */ + +/* if splitting of data transfer is enabled, this callback is called before + download of an individual chunk started. Note that parameter "remains" works + only for FTP wildcard downloading (for now), otherwise is not used */ +typedef long (*curl_chunk_bgn_callback)(const void *transfer_info, + void *ptr, + int remains); + +/* return codes for CURLOPT_CHUNK_END_FUNCTION */ +#define CURL_CHUNK_END_FUNC_OK 0 +#define CURL_CHUNK_END_FUNC_FAIL 1 /* tell the lib to end the task */ + +/* If splitting of data transfer is enabled this callback is called after + download of an individual chunk finished. + Note! After this callback was set then it have to be called FOR ALL chunks. + Even if downloading of this chunk was skipped in CHUNK_BGN_FUNC. + This is the reason why we don't need "transfer_info" parameter in this + callback and we are not interested in "remains" parameter too. */ +typedef long (*curl_chunk_end_callback)(void *ptr); + +/* return codes for FNMATCHFUNCTION */ +#define CURL_FNMATCHFUNC_MATCH 0 /* string corresponds to the pattern */ +#define CURL_FNMATCHFUNC_NOMATCH 1 /* pattern doesn't match the string */ +#define CURL_FNMATCHFUNC_FAIL 2 /* an error occurred */ + +/* callback type for wildcard downloading pattern matching. If the + string matches the pattern, return CURL_FNMATCHFUNC_MATCH value, etc. */ +typedef int (*curl_fnmatch_callback)(void *ptr, + const char *pattern, + const char *string); + +/* These are the return codes for the seek callbacks */ +#define CURL_SEEKFUNC_OK 0 +#define CURL_SEEKFUNC_FAIL 1 /* fail the entire transfer */ +#define CURL_SEEKFUNC_CANTSEEK 2 /* tell libcurl seeking can't be done, so + libcurl might try other means instead */ +typedef int (*curl_seek_callback)(void *instream, + curl_off_t offset, + int origin); /* 'whence' */ + +/* This is a return code for the read callback that, when returned, will + signal libcurl to immediately abort the current transfer. */ +#define CURL_READFUNC_ABORT 0x10000000 +/* This is a return code for the read callback that, when returned, will + signal libcurl to pause sending data on the current transfer. */ +#define CURL_READFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_read_callback)(char *buffer, + size_t size, + size_t nitems, + void *instream); + +typedef enum { + CURLSOCKTYPE_IPCXN, /* socket created for a specific IP connection */ + CURLSOCKTYPE_ACCEPT, /* socket created by accept() call */ + CURLSOCKTYPE_LAST /* never use */ +} curlsocktype; + +/* The return code from the sockopt_callback can signal information back + to libcurl: */ +#define CURL_SOCKOPT_OK 0 +#define CURL_SOCKOPT_ERROR 1 /* causes libcurl to abort and return + CURLE_ABORTED_BY_CALLBACK */ +#define CURL_SOCKOPT_ALREADY_CONNECTED 2 + +typedef int (*curl_sockopt_callback)(void *clientp, + curl_socket_t curlfd, + curlsocktype purpose); + +struct curl_sockaddr { + int family; + int socktype; + int protocol; + unsigned int addrlen; /* addrlen was a socklen_t type before 7.18.0 but it + turned really ugly and painful on the systems that + lack this type */ + struct sockaddr addr; +}; + +typedef curl_socket_t +(*curl_opensocket_callback)(void *clientp, + curlsocktype purpose, + struct curl_sockaddr *address); + +typedef int +(*curl_closesocket_callback)(void *clientp, curl_socket_t item); + +typedef enum { + CURLIOE_OK, /* I/O operation successful */ + CURLIOE_UNKNOWNCMD, /* command was unknown to callback */ + CURLIOE_FAILRESTART, /* failed to restart the read */ + CURLIOE_LAST /* never use */ +} curlioerr; + +typedef enum { + CURLIOCMD_NOP, /* no operation */ + CURLIOCMD_RESTARTREAD, /* restart the read stream from start */ + CURLIOCMD_LAST /* never use */ +} curliocmd; + +typedef curlioerr (*curl_ioctl_callback)(CURL *handle, + int cmd, + void *clientp); + +/* + * The following typedef's are signatures of malloc, free, realloc, strdup and + * calloc respectively. Function pointers of these types can be passed to the + * curl_global_init_mem() function to set user defined memory management + * callback routines. + */ +typedef void *(*curl_malloc_callback)(size_t size); +typedef void (*curl_free_callback)(void *ptr); +typedef void *(*curl_realloc_callback)(void *ptr, size_t size); +typedef char *(*curl_strdup_callback)(const char *str); +typedef void *(*curl_calloc_callback)(size_t nmemb, size_t size); + +/* the kind of data that is passed to information_callback*/ +typedef enum { + CURLINFO_TEXT = 0, + CURLINFO_HEADER_IN, /* 1 */ + CURLINFO_HEADER_OUT, /* 2 */ + CURLINFO_DATA_IN, /* 3 */ + CURLINFO_DATA_OUT, /* 4 */ + CURLINFO_SSL_DATA_IN, /* 5 */ + CURLINFO_SSL_DATA_OUT, /* 6 */ + CURLINFO_END +} curl_infotype; + +typedef int (*curl_debug_callback) + (CURL *handle, /* the handle/transfer this concerns */ + curl_infotype type, /* what kind of data */ + char *data, /* points to the data */ + size_t size, /* size of the data pointed to */ + void *userptr); /* whatever the user please */ + +/* All possible error codes from all sorts of curl functions. Future versions + may return other values, stay prepared. + + Always add new return codes last. Never *EVER* remove any. The return + codes must remain the same! + */ + +typedef enum { + CURLE_OK = 0, + CURLE_UNSUPPORTED_PROTOCOL, /* 1 */ + CURLE_FAILED_INIT, /* 2 */ + CURLE_URL_MALFORMAT, /* 3 */ + CURLE_NOT_BUILT_IN, /* 4 - [was obsoleted in August 2007 for + 7.17.0, reused in April 2011 for 7.21.5] */ + CURLE_COULDNT_RESOLVE_PROXY, /* 5 */ + CURLE_COULDNT_RESOLVE_HOST, /* 6 */ + CURLE_COULDNT_CONNECT, /* 7 */ + CURLE_FTP_WEIRD_SERVER_REPLY, /* 8 */ + CURLE_REMOTE_ACCESS_DENIED, /* 9 a service was denied by the server + due to lack of access - when login fails + this is not returned. */ + CURLE_FTP_ACCEPT_FAILED, /* 10 - [was obsoleted in April 2006 for + 7.15.4, reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASS_REPLY, /* 11 */ + CURLE_FTP_ACCEPT_TIMEOUT, /* 12 - timeout occurred accepting server + [was obsoleted in August 2007 for 7.17.0, + reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASV_REPLY, /* 13 */ + CURLE_FTP_WEIRD_227_FORMAT, /* 14 */ + CURLE_FTP_CANT_GET_HOST, /* 15 */ + CURLE_HTTP2, /* 16 - A problem in the http2 framing layer. + [was obsoleted in August 2007 for 7.17.0, + reused in July 2014 for 7.38.0] */ + CURLE_FTP_COULDNT_SET_TYPE, /* 17 */ + CURLE_PARTIAL_FILE, /* 18 */ + CURLE_FTP_COULDNT_RETR_FILE, /* 19 */ + CURLE_OBSOLETE20, /* 20 - NOT USED */ + CURLE_QUOTE_ERROR, /* 21 - quote command failure */ + CURLE_HTTP_RETURNED_ERROR, /* 22 */ + CURLE_WRITE_ERROR, /* 23 */ + CURLE_OBSOLETE24, /* 24 - NOT USED */ + CURLE_UPLOAD_FAILED, /* 25 - failed upload "command" */ + CURLE_READ_ERROR, /* 26 - couldn't open/read from file */ + CURLE_OUT_OF_MEMORY, /* 27 */ + /* Note: CURLE_OUT_OF_MEMORY may sometimes indicate a conversion error + instead of a memory allocation error if CURL_DOES_CONVERSIONS + is defined + */ + CURLE_OPERATION_TIMEDOUT, /* 28 - the timeout time was reached */ + CURLE_OBSOLETE29, /* 29 - NOT USED */ + CURLE_FTP_PORT_FAILED, /* 30 - FTP PORT operation failed */ + CURLE_FTP_COULDNT_USE_REST, /* 31 - the REST command failed */ + CURLE_OBSOLETE32, /* 32 - NOT USED */ + CURLE_RANGE_ERROR, /* 33 - RANGE "command" didn't work */ + CURLE_HTTP_POST_ERROR, /* 34 */ + CURLE_SSL_CONNECT_ERROR, /* 35 - wrong when connecting with SSL */ + CURLE_BAD_DOWNLOAD_RESUME, /* 36 - couldn't resume download */ + CURLE_FILE_COULDNT_READ_FILE, /* 37 */ + CURLE_LDAP_CANNOT_BIND, /* 38 */ + CURLE_LDAP_SEARCH_FAILED, /* 39 */ + CURLE_OBSOLETE40, /* 40 - NOT USED */ + CURLE_FUNCTION_NOT_FOUND, /* 41 */ + CURLE_ABORTED_BY_CALLBACK, /* 42 */ + CURLE_BAD_FUNCTION_ARGUMENT, /* 43 */ + CURLE_OBSOLETE44, /* 44 - NOT USED */ + CURLE_INTERFACE_FAILED, /* 45 - CURLOPT_INTERFACE failed */ + CURLE_OBSOLETE46, /* 46 - NOT USED */ + CURLE_TOO_MANY_REDIRECTS , /* 47 - catch endless re-direct loops */ + CURLE_UNKNOWN_OPTION, /* 48 - User specified an unknown option */ + CURLE_TELNET_OPTION_SYNTAX , /* 49 - Malformed telnet option */ + CURLE_OBSOLETE50, /* 50 - NOT USED */ + CURLE_PEER_FAILED_VERIFICATION, /* 51 - peer's certificate or fingerprint + wasn't verified fine */ + CURLE_GOT_NOTHING, /* 52 - when this is a specific error */ + CURLE_SSL_ENGINE_NOTFOUND, /* 53 - SSL crypto engine not found */ + CURLE_SSL_ENGINE_SETFAILED, /* 54 - can not set SSL crypto engine as + default */ + CURLE_SEND_ERROR, /* 55 - failed sending network data */ + CURLE_RECV_ERROR, /* 56 - failure in receiving network data */ + CURLE_OBSOLETE57, /* 57 - NOT IN USE */ + CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */ + CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */ + CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */ + CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized/bad encoding */ + CURLE_LDAP_INVALID_URL, /* 62 - Invalid LDAP URL */ + CURLE_FILESIZE_EXCEEDED, /* 63 - Maximum file size exceeded */ + CURLE_USE_SSL_FAILED, /* 64 - Requested FTP SSL level failed */ + CURLE_SEND_FAIL_REWIND, /* 65 - Sending the data requires a rewind + that failed */ + CURLE_SSL_ENGINE_INITFAILED, /* 66 - failed to initialise ENGINE */ + CURLE_LOGIN_DENIED, /* 67 - user, password or similar was not + accepted and we failed to login */ + CURLE_TFTP_NOTFOUND, /* 68 - file not found on server */ + CURLE_TFTP_PERM, /* 69 - permission problem on server */ + CURLE_REMOTE_DISK_FULL, /* 70 - out of disk space on server */ + CURLE_TFTP_ILLEGAL, /* 71 - Illegal TFTP operation */ + CURLE_TFTP_UNKNOWNID, /* 72 - Unknown transfer ID */ + CURLE_REMOTE_FILE_EXISTS, /* 73 - File already exists */ + CURLE_TFTP_NOSUCHUSER, /* 74 - No such user */ + CURLE_CONV_FAILED, /* 75 - conversion failed */ + CURLE_CONV_REQD, /* 76 - caller must register conversion + callbacks using curl_easy_setopt options + CURLOPT_CONV_FROM_NETWORK_FUNCTION, + CURLOPT_CONV_TO_NETWORK_FUNCTION, and + CURLOPT_CONV_FROM_UTF8_FUNCTION */ + CURLE_SSL_CACERT_BADFILE, /* 77 - could not load CACERT file, missing + or wrong format */ + CURLE_REMOTE_FILE_NOT_FOUND, /* 78 - remote file not found */ + CURLE_SSH, /* 79 - error from the SSH layer, somewhat + generic so the error message will be of + interest when this has happened */ + + CURLE_SSL_SHUTDOWN_FAILED, /* 80 - Failed to shut down the SSL + connection */ + CURLE_AGAIN, /* 81 - socket is not ready for send/recv, + wait till it's ready and try again (Added + in 7.18.2) */ + CURLE_SSL_CRL_BADFILE, /* 82 - could not load CRL file, missing or + wrong format (Added in 7.19.0) */ + CURLE_SSL_ISSUER_ERROR, /* 83 - Issuer check failed. (Added in + 7.19.0) */ + CURLE_FTP_PRET_FAILED, /* 84 - a PRET command failed */ + CURLE_RTSP_CSEQ_ERROR, /* 85 - mismatch of RTSP CSeq numbers */ + CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Ids */ + CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ + CURLE_CHUNK_FAILED, /* 88 - chunk callback reported error */ + CURLE_NO_CONNECTION_AVAILABLE, /* 89 - No connection available, the + session will be queued */ + CURLE_SSL_PINNEDPUBKEYNOTMATCH, /* 90 - specified pinned public key did not + match */ + CURLE_SSL_INVALIDCERTSTATUS, /* 91 - invalid certificate status */ + CURL_LAST /* never use! */ +} CURLcode; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Previously obsolete error code re-used in 7.38.0 */ +#define CURLE_OBSOLETE16 CURLE_HTTP2 + +/* Previously obsolete error codes re-used in 7.24.0 */ +#define CURLE_OBSOLETE10 CURLE_FTP_ACCEPT_FAILED +#define CURLE_OBSOLETE12 CURLE_FTP_ACCEPT_TIMEOUT + +/* compatibility with older names */ +#define CURLOPT_ENCODING CURLOPT_ACCEPT_ENCODING + +/* The following were added in 7.21.5, April 2011 */ +#define CURLE_UNKNOWN_TELNET_OPTION CURLE_UNKNOWN_OPTION + +/* The following were added in 7.17.1 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_SSL_PEER_CERTIFICATE CURLE_PEER_FAILED_VERIFICATION + +/* The following were added in 7.17.0 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_OBSOLETE CURLE_OBSOLETE50 /* no one should be using this! */ +#define CURLE_BAD_PASSWORD_ENTERED CURLE_OBSOLETE46 +#define CURLE_BAD_CALLING_ORDER CURLE_OBSOLETE44 +#define CURLE_FTP_USER_PASSWORD_INCORRECT CURLE_OBSOLETE10 +#define CURLE_FTP_CANT_RECONNECT CURLE_OBSOLETE16 +#define CURLE_FTP_COULDNT_GET_SIZE CURLE_OBSOLETE32 +#define CURLE_FTP_COULDNT_SET_ASCII CURLE_OBSOLETE29 +#define CURLE_FTP_WEIRD_USER_REPLY CURLE_OBSOLETE12 +#define CURLE_FTP_WRITE_ERROR CURLE_OBSOLETE20 +#define CURLE_LIBRARY_NOT_FOUND CURLE_OBSOLETE40 +#define CURLE_MALFORMAT_USER CURLE_OBSOLETE24 +#define CURLE_SHARE_IN_USE CURLE_OBSOLETE57 +#define CURLE_URL_MALFORMAT_USER CURLE_NOT_BUILT_IN + +#define CURLE_FTP_ACCESS_DENIED CURLE_REMOTE_ACCESS_DENIED +#define CURLE_FTP_COULDNT_SET_BINARY CURLE_FTP_COULDNT_SET_TYPE +#define CURLE_FTP_QUOTE_ERROR CURLE_QUOTE_ERROR +#define CURLE_TFTP_DISKFULL CURLE_REMOTE_DISK_FULL +#define CURLE_TFTP_EXISTS CURLE_REMOTE_FILE_EXISTS +#define CURLE_HTTP_RANGE_ERROR CURLE_RANGE_ERROR +#define CURLE_FTP_SSL_FAILED CURLE_USE_SSL_FAILED + +/* The following were added earlier */ + +#define CURLE_OPERATION_TIMEOUTED CURLE_OPERATION_TIMEDOUT + +#define CURLE_HTTP_NOT_FOUND CURLE_HTTP_RETURNED_ERROR +#define CURLE_HTTP_PORT_FAILED CURLE_INTERFACE_FAILED +#define CURLE_FTP_COULDNT_STOR_FILE CURLE_UPLOAD_FAILED + +#define CURLE_FTP_PARTIAL_FILE CURLE_PARTIAL_FILE +#define CURLE_FTP_BAD_DOWNLOAD_RESUME CURLE_BAD_DOWNLOAD_RESUME + +/* This was the error code 50 in 7.7.3 and a few earlier versions, this + is no longer used by libcurl but is instead #defined here only to not + make programs break */ +#define CURLE_ALREADY_COMPLETE 99999 + +/* Provide defines for really old option names */ +#define CURLOPT_FILE CURLOPT_WRITEDATA /* name changed in 7.9.7 */ +#define CURLOPT_INFILE CURLOPT_READDATA /* name changed in 7.9.7 */ +#define CURLOPT_WRITEHEADER CURLOPT_HEADERDATA + +/* Since long deprecated options with no code in the lib that does anything + with them. */ +#define CURLOPT_WRITEINFO CURLOPT_OBSOLETE40 +#define CURLOPT_CLOSEPOLICY CURLOPT_OBSOLETE72 + +#endif /*!CURL_NO_OLDIES*/ + +/* This prototype applies to all conversion callbacks */ +typedef CURLcode (*curl_conv_callback)(char *buffer, size_t length); + +typedef CURLcode (*curl_ssl_ctx_callback)(CURL *curl, /* easy handle */ + void *ssl_ctx, /* actually an + OpenSSL SSL_CTX */ + void *userptr); + +typedef enum { + CURLPROXY_HTTP = 0, /* added in 7.10, new in 7.19.4 default is to use + CONNECT HTTP/1.1 */ + CURLPROXY_HTTP_1_0 = 1, /* added in 7.19.4, force to use CONNECT + HTTP/1.0 */ + CURLPROXY_SOCKS4 = 4, /* support added in 7.15.2, enum existed already + in 7.10 */ + CURLPROXY_SOCKS5 = 5, /* added in 7.10 */ + CURLPROXY_SOCKS4A = 6, /* added in 7.18.0 */ + CURLPROXY_SOCKS5_HOSTNAME = 7 /* Use the SOCKS5 protocol but pass along the + host name rather than the IP address. added + in 7.18.0 */ +} curl_proxytype; /* this enum was added in 7.10 */ + +/* + * Bitmasks for CURLOPT_HTTPAUTH and CURLOPT_PROXYAUTH options: + * + * CURLAUTH_NONE - No HTTP authentication + * CURLAUTH_BASIC - HTTP Basic authentication (default) + * CURLAUTH_DIGEST - HTTP Digest authentication + * CURLAUTH_NEGOTIATE - HTTP Negotiate (SPNEGO) authentication + * CURLAUTH_GSSNEGOTIATE - Alias for CURLAUTH_NEGOTIATE (deprecated) + * CURLAUTH_NTLM - HTTP NTLM authentication + * CURLAUTH_DIGEST_IE - HTTP Digest authentication with IE flavour + * CURLAUTH_NTLM_WB - HTTP NTLM authentication delegated to winbind helper + * CURLAUTH_ONLY - Use together with a single other type to force no + * authentication or just that single type + * CURLAUTH_ANY - All fine types set + * CURLAUTH_ANYSAFE - All fine types except Basic + */ + +#define CURLAUTH_NONE ((unsigned long)0) +#define CURLAUTH_BASIC (((unsigned long)1)<<0) +#define CURLAUTH_DIGEST (((unsigned long)1)<<1) +#define CURLAUTH_NEGOTIATE (((unsigned long)1)<<2) +/* Deprecated since the advent of CURLAUTH_NEGOTIATE */ +#define CURLAUTH_GSSNEGOTIATE CURLAUTH_NEGOTIATE +#define CURLAUTH_NTLM (((unsigned long)1)<<3) +#define CURLAUTH_DIGEST_IE (((unsigned long)1)<<4) +#define CURLAUTH_NTLM_WB (((unsigned long)1)<<5) +#define CURLAUTH_ONLY (((unsigned long)1)<<31) +#define CURLAUTH_ANY (~CURLAUTH_DIGEST_IE) +#define CURLAUTH_ANYSAFE (~(CURLAUTH_BASIC|CURLAUTH_DIGEST_IE)) + +#define CURLSSH_AUTH_ANY ~0 /* all types supported by the server */ +#define CURLSSH_AUTH_NONE 0 /* none allowed, silly but complete */ +#define CURLSSH_AUTH_PUBLICKEY (1<<0) /* public/private key files */ +#define CURLSSH_AUTH_PASSWORD (1<<1) /* password */ +#define CURLSSH_AUTH_HOST (1<<2) /* host key files */ +#define CURLSSH_AUTH_KEYBOARD (1<<3) /* keyboard interactive */ +#define CURLSSH_AUTH_AGENT (1<<4) /* agent (ssh-agent, pageant...) */ +#define CURLSSH_AUTH_DEFAULT CURLSSH_AUTH_ANY + +#define CURLGSSAPI_DELEGATION_NONE 0 /* no delegation (default) */ +#define CURLGSSAPI_DELEGATION_POLICY_FLAG (1<<0) /* if permitted by policy */ +#define CURLGSSAPI_DELEGATION_FLAG (1<<1) /* delegate always */ + +#define CURL_ERROR_SIZE 256 + +enum curl_khtype { + CURLKHTYPE_UNKNOWN, + CURLKHTYPE_RSA1, + CURLKHTYPE_RSA, + CURLKHTYPE_DSS +}; + +struct curl_khkey { + const char *key; /* points to a zero-terminated string encoded with base64 + if len is zero, otherwise to the "raw" data */ + size_t len; + enum curl_khtype keytype; +}; + +/* this is the set of return values expected from the curl_sshkeycallback + callback */ +enum curl_khstat { + CURLKHSTAT_FINE_ADD_TO_FILE, + CURLKHSTAT_FINE, + CURLKHSTAT_REJECT, /* reject the connection, return an error */ + CURLKHSTAT_DEFER, /* do not accept it, but we can't answer right now so + this causes a CURLE_DEFER error but otherwise the + connection will be left intact etc */ + CURLKHSTAT_LAST /* not for use, only a marker for last-in-list */ +}; + +/* this is the set of status codes pass in to the callback */ +enum curl_khmatch { + CURLKHMATCH_OK, /* match */ + CURLKHMATCH_MISMATCH, /* host found, key mismatch! */ + CURLKHMATCH_MISSING, /* no matching host/key found */ + CURLKHMATCH_LAST /* not for use, only a marker for last-in-list */ +}; + +typedef int + (*curl_sshkeycallback) (CURL *easy, /* easy handle */ + const struct curl_khkey *knownkey, /* known */ + const struct curl_khkey *foundkey, /* found */ + enum curl_khmatch, /* libcurl's view on the keys */ + void *clientp); /* custom pointer passed from app */ + +/* parameter for the CURLOPT_USE_SSL option */ +typedef enum { + CURLUSESSL_NONE, /* do not attempt to use SSL */ + CURLUSESSL_TRY, /* try using SSL, proceed anyway otherwise */ + CURLUSESSL_CONTROL, /* SSL for the control connection or fail */ + CURLUSESSL_ALL, /* SSL for all communication or fail */ + CURLUSESSL_LAST /* not an option, never use */ +} curl_usessl; + +/* Definition of bits for the CURLOPT_SSL_OPTIONS argument: */ + +/* - ALLOW_BEAST tells libcurl to allow the BEAST SSL vulnerability in the + name of improving interoperability with older servers. Some SSL libraries + have introduced work-arounds for this flaw but those work-arounds sometimes + make the SSL communication fail. To regain functionality with those broken + servers, a user can this way allow the vulnerability back. */ +#define CURLSSLOPT_ALLOW_BEAST (1<<0) + +/* - NO_REVOKE tells libcurl to disable certificate revocation checks for those + SSL backends where such behavior is present. */ +#define CURLSSLOPT_NO_REVOKE (1<<1) + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2009 */ + +#define CURLFTPSSL_NONE CURLUSESSL_NONE +#define CURLFTPSSL_TRY CURLUSESSL_TRY +#define CURLFTPSSL_CONTROL CURLUSESSL_CONTROL +#define CURLFTPSSL_ALL CURLUSESSL_ALL +#define CURLFTPSSL_LAST CURLUSESSL_LAST +#define curl_ftpssl curl_usessl +#endif /*!CURL_NO_OLDIES*/ + +/* parameter for the CURLOPT_FTP_SSL_CCC option */ +typedef enum { + CURLFTPSSL_CCC_NONE, /* do not send CCC */ + CURLFTPSSL_CCC_PASSIVE, /* Let the server initiate the shutdown */ + CURLFTPSSL_CCC_ACTIVE, /* Initiate the shutdown */ + CURLFTPSSL_CCC_LAST /* not an option, never use */ +} curl_ftpccc; + +/* parameter for the CURLOPT_FTPSSLAUTH option */ +typedef enum { + CURLFTPAUTH_DEFAULT, /* let libcurl decide */ + CURLFTPAUTH_SSL, /* use "AUTH SSL" */ + CURLFTPAUTH_TLS, /* use "AUTH TLS" */ + CURLFTPAUTH_LAST /* not an option, never use */ +} curl_ftpauth; + +/* parameter for the CURLOPT_FTP_CREATE_MISSING_DIRS option */ +typedef enum { + CURLFTP_CREATE_DIR_NONE, /* do NOT create missing dirs! */ + CURLFTP_CREATE_DIR, /* (FTP/SFTP) if CWD fails, try MKD and then CWD + again if MKD succeeded, for SFTP this does + similar magic */ + CURLFTP_CREATE_DIR_RETRY, /* (FTP only) if CWD fails, try MKD and then CWD + again even if MKD failed! */ + CURLFTP_CREATE_DIR_LAST /* not an option, never use */ +} curl_ftpcreatedir; + +/* parameter for the CURLOPT_FTP_FILEMETHOD option */ +typedef enum { + CURLFTPMETHOD_DEFAULT, /* let libcurl pick */ + CURLFTPMETHOD_MULTICWD, /* single CWD operation for each path part */ + CURLFTPMETHOD_NOCWD, /* no CWD at all */ + CURLFTPMETHOD_SINGLECWD, /* one CWD to full dir, then work on file */ + CURLFTPMETHOD_LAST /* not an option, never use */ +} curl_ftpmethod; + +/* bitmask defines for CURLOPT_HEADEROPT */ +#define CURLHEADER_UNIFIED 0 +#define CURLHEADER_SEPARATE (1<<0) + +/* CURLPROTO_ defines are for the CURLOPT_*PROTOCOLS options */ +#define CURLPROTO_HTTP (1<<0) +#define CURLPROTO_HTTPS (1<<1) +#define CURLPROTO_FTP (1<<2) +#define CURLPROTO_FTPS (1<<3) +#define CURLPROTO_SCP (1<<4) +#define CURLPROTO_SFTP (1<<5) +#define CURLPROTO_TELNET (1<<6) +#define CURLPROTO_LDAP (1<<7) +#define CURLPROTO_LDAPS (1<<8) +#define CURLPROTO_DICT (1<<9) +#define CURLPROTO_FILE (1<<10) +#define CURLPROTO_TFTP (1<<11) +#define CURLPROTO_IMAP (1<<12) +#define CURLPROTO_IMAPS (1<<13) +#define CURLPROTO_POP3 (1<<14) +#define CURLPROTO_POP3S (1<<15) +#define CURLPROTO_SMTP (1<<16) +#define CURLPROTO_SMTPS (1<<17) +#define CURLPROTO_RTSP (1<<18) +#define CURLPROTO_RTMP (1<<19) +#define CURLPROTO_RTMPT (1<<20) +#define CURLPROTO_RTMPE (1<<21) +#define CURLPROTO_RTMPTE (1<<22) +#define CURLPROTO_RTMPS (1<<23) +#define CURLPROTO_RTMPTS (1<<24) +#define CURLPROTO_GOPHER (1<<25) +#define CURLPROTO_SMB (1<<26) +#define CURLPROTO_SMBS (1<<27) +#define CURLPROTO_ALL (~0) /* enable everything */ + +/* long may be 32 or 64 bits, but we should never depend on anything else + but 32 */ +#define CURLOPTTYPE_LONG 0 +#define CURLOPTTYPE_OBJECTPOINT 10000 +#define CURLOPTTYPE_STRINGPOINT 10000 +#define CURLOPTTYPE_FUNCTIONPOINT 20000 +#define CURLOPTTYPE_OFF_T 30000 + +/* *STRINGPOINT is an alias for OBJECTPOINT to allow tools to extract the + string options from the header file */ + +/* name is uppercase CURLOPT_, + type is one of the defined CURLOPTTYPE_ + number is unique identifier */ +#ifdef CINIT +#undef CINIT +#endif + +#ifdef CURL_ISOCPP +#define CINIT(na,t,nu) CURLOPT_ ## na = CURLOPTTYPE_ ## t + nu +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define STRINGPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLOPT_/**/name = type + number +#endif + +/* + * This macro-mania below setups the CURLOPT_[what] enum, to be used with + * curl_easy_setopt(). The first argument in the CINIT() macro is the [what] + * word. + */ + +typedef enum { + /* This is the FILE * or void * the regular output should be written to. */ + CINIT(WRITEDATA, OBJECTPOINT, 1), + + /* The full URL to get/put */ + CINIT(URL, STRINGPOINT, 2), + + /* Port number to connect to, if other than default. */ + CINIT(PORT, LONG, 3), + + /* Name of proxy to use. */ + CINIT(PROXY, STRINGPOINT, 4), + + /* "user:password;options" to use when fetching. */ + CINIT(USERPWD, STRINGPOINT, 5), + + /* "user:password" to use with proxy. */ + CINIT(PROXYUSERPWD, STRINGPOINT, 6), + + /* Range to get, specified as an ASCII string. */ + CINIT(RANGE, STRINGPOINT, 7), + + /* not used */ + + /* Specified file stream to upload from (use as input): */ + CINIT(READDATA, OBJECTPOINT, 9), + + /* Buffer to receive error messages in, must be at least CURL_ERROR_SIZE + * bytes big. If this is not used, error messages go to stderr instead: */ + CINIT(ERRORBUFFER, OBJECTPOINT, 10), + + /* Function that will be called to store the output (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11), + + /* Function that will be called to read the input (instead of fread). The + * parameters will use fread() syntax, make sure to follow them. */ + CINIT(READFUNCTION, FUNCTIONPOINT, 12), + + /* Time-out the read operation after this amount of seconds */ + CINIT(TIMEOUT, LONG, 13), + + /* If the CURLOPT_INFILE is used, this can be used to inform libcurl about + * how large the file being sent really is. That allows better error + * checking and better verifies that the upload was successful. -1 means + * unknown size. + * + * For large file support, there is also a _LARGE version of the key + * which takes an off_t type, allowing platforms with larger off_t + * sizes to handle larger files. See below for INFILESIZE_LARGE. + */ + CINIT(INFILESIZE, LONG, 14), + + /* POST static input fields. */ + CINIT(POSTFIELDS, OBJECTPOINT, 15), + + /* Set the referrer page (needed by some CGIs) */ + CINIT(REFERER, STRINGPOINT, 16), + + /* Set the FTP PORT string (interface name, named or numerical IP address) + Use i.e '-' to use default address. */ + CINIT(FTPPORT, STRINGPOINT, 17), + + /* Set the User-Agent string (examined by some CGIs) */ + CINIT(USERAGENT, STRINGPOINT, 18), + + /* If the download receives less than "low speed limit" bytes/second + * during "low speed time" seconds, the operations is aborted. + * You could i.e if you have a pretty high speed connection, abort if + * it is less than 2000 bytes/sec during 20 seconds. + */ + + /* Set the "low speed limit" */ + CINIT(LOW_SPEED_LIMIT, LONG, 19), + + /* Set the "low speed time" */ + CINIT(LOW_SPEED_TIME, LONG, 20), + + /* Set the continuation offset. + * + * Note there is also a _LARGE version of this key which uses + * off_t types, allowing for large file offsets on platforms which + * use larger-than-32-bit off_t's. Look below for RESUME_FROM_LARGE. + */ + CINIT(RESUME_FROM, LONG, 21), + + /* Set cookie in request: */ + CINIT(COOKIE, STRINGPOINT, 22), + + /* This points to a linked list of headers, struct curl_slist kind. This + list is also used for RTSP (in spite of its name) */ + CINIT(HTTPHEADER, OBJECTPOINT, 23), + + /* This points to a linked list of post entries, struct curl_httppost */ + CINIT(HTTPPOST, OBJECTPOINT, 24), + + /* name of the file keeping your private SSL-certificate */ + CINIT(SSLCERT, STRINGPOINT, 25), + + /* password for the SSL or SSH private key */ + CINIT(KEYPASSWD, STRINGPOINT, 26), + + /* send TYPE parameter? */ + CINIT(CRLF, LONG, 27), + + /* send linked-list of QUOTE commands */ + CINIT(QUOTE, OBJECTPOINT, 28), + + /* send FILE * or void * to store headers to, if you use a callback it + is simply passed to the callback unmodified */ + CINIT(HEADERDATA, OBJECTPOINT, 29), + + /* point to a file to read the initial cookies from, also enables + "cookie awareness" */ + CINIT(COOKIEFILE, STRINGPOINT, 31), + + /* What version to specifically try to use. + See CURL_SSLVERSION defines below. */ + CINIT(SSLVERSION, LONG, 32), + + /* What kind of HTTP time condition to use, see defines */ + CINIT(TIMECONDITION, LONG, 33), + + /* Time to use with the above condition. Specified in number of seconds + since 1 Jan 1970 */ + CINIT(TIMEVALUE, LONG, 34), + + /* 35 = OBSOLETE */ + + /* Custom request, for customizing the get command like + HTTP: DELETE, TRACE and others + FTP: to use a different list command + */ + CINIT(CUSTOMREQUEST, STRINGPOINT, 36), + + /* FILE handle to use instead of stderr */ + CINIT(STDERR, OBJECTPOINT, 37), + + /* 38 is not used */ + + /* send linked-list of post-transfer QUOTE commands */ + CINIT(POSTQUOTE, OBJECTPOINT, 39), + + CINIT(OBSOLETE40, OBJECTPOINT, 40), /* OBSOLETE, do not use! */ + + CINIT(VERBOSE, LONG, 41), /* talk a lot */ + CINIT(HEADER, LONG, 42), /* throw the header out too */ + CINIT(NOPROGRESS, LONG, 43), /* shut off the progress meter */ + CINIT(NOBODY, LONG, 44), /* use HEAD to get http document */ + CINIT(FAILONERROR, LONG, 45), /* no output on http error codes >= 400 */ + CINIT(UPLOAD, LONG, 46), /* this is an upload */ + CINIT(POST, LONG, 47), /* HTTP POST method */ + CINIT(DIRLISTONLY, LONG, 48), /* bare names when listing directories */ + + CINIT(APPEND, LONG, 50), /* Append instead of overwrite on upload! */ + + /* Specify whether to read the user+password from the .netrc or the URL. + * This must be one of the CURL_NETRC_* enums below. */ + CINIT(NETRC, LONG, 51), + + CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */ + + CINIT(TRANSFERTEXT, LONG, 53), /* transfer data in text/ASCII format */ + CINIT(PUT, LONG, 54), /* HTTP PUT */ + + /* 55 = OBSOLETE */ + + /* DEPRECATED + * Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_progress_callback + * prototype defines. */ + CINIT(PROGRESSFUNCTION, FUNCTIONPOINT, 56), + + /* Data passed to the CURLOPT_PROGRESSFUNCTION and CURLOPT_XFERINFOFUNCTION + callbacks */ + CINIT(PROGRESSDATA, OBJECTPOINT, 57), +#define CURLOPT_XFERINFODATA CURLOPT_PROGRESSDATA + + /* We want the referrer field set automatically when following locations */ + CINIT(AUTOREFERER, LONG, 58), + + /* Port of the proxy, can be set in the proxy string as well with: + "[host]:[port]" */ + CINIT(PROXYPORT, LONG, 59), + + /* size of the POST input data, if strlen() is not good to use */ + CINIT(POSTFIELDSIZE, LONG, 60), + + /* tunnel non-http operations through a HTTP proxy */ + CINIT(HTTPPROXYTUNNEL, LONG, 61), + + /* Set the interface string to use as outgoing network interface */ + CINIT(INTERFACE, STRINGPOINT, 62), + + /* Set the krb4/5 security level, this also enables krb4/5 awareness. This + * is a string, 'clear', 'safe', 'confidential' or 'private'. If the string + * is set but doesn't match one of these, 'private' will be used. */ + CINIT(KRBLEVEL, STRINGPOINT, 63), + + /* Set if we should verify the peer in ssl handshake, set 1 to verify. */ + CINIT(SSL_VERIFYPEER, LONG, 64), + + /* The CApath or CAfile used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAINFO, STRINGPOINT, 65), + + /* 66 = OBSOLETE */ + /* 67 = OBSOLETE */ + + /* Maximum number of http redirects to follow */ + CINIT(MAXREDIRS, LONG, 68), + + /* Pass a long set to 1 to get the date of the requested document (if + possible)! Pass a zero to shut it off. */ + CINIT(FILETIME, LONG, 69), + + /* This points to a linked list of telnet options */ + CINIT(TELNETOPTIONS, OBJECTPOINT, 70), + + /* Max amount of cached alive connections */ + CINIT(MAXCONNECTS, LONG, 71), + + CINIT(OBSOLETE72, LONG, 72), /* OBSOLETE, do not use! */ + + /* 73 = OBSOLETE */ + + /* Set to explicitly use a new connection for the upcoming transfer. + Do not use this unless you're absolutely sure of this, as it makes the + operation slower and is less friendly for the network. */ + CINIT(FRESH_CONNECT, LONG, 74), + + /* Set to explicitly forbid the upcoming transfer's connection to be re-used + when done. Do not use this unless you're absolutely sure of this, as it + makes the operation slower and is less friendly for the network. */ + CINIT(FORBID_REUSE, LONG, 75), + + /* Set to a file name that contains random data for libcurl to use to + seed the random engine when doing SSL connects. */ + CINIT(RANDOM_FILE, STRINGPOINT, 76), + + /* Set to the Entropy Gathering Daemon socket pathname */ + CINIT(EGDSOCKET, STRINGPOINT, 77), + + /* Time-out connect operations after this amount of seconds, if connects are + OK within this time, then fine... This only aborts the connect phase. */ + CINIT(CONNECTTIMEOUT, LONG, 78), + + /* Function that will be called to store headers (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(HEADERFUNCTION, FUNCTIONPOINT, 79), + + /* Set this to force the HTTP request to get back to GET. Only really usable + if POST, PUT or a custom request have been used first. + */ + CINIT(HTTPGET, LONG, 80), + + /* Set if we should verify the Common name from the peer certificate in ssl + * handshake, set 1 to check existence, 2 to ensure that it matches the + * provided hostname. */ + CINIT(SSL_VERIFYHOST, LONG, 81), + + /* Specify which file name to write all known cookies in after completed + operation. Set file name to "-" (dash) to make it go to stdout. */ + CINIT(COOKIEJAR, STRINGPOINT, 82), + + /* Specify which SSL ciphers to use */ + CINIT(SSL_CIPHER_LIST, STRINGPOINT, 83), + + /* Specify which HTTP version to use! This must be set to one of the + CURL_HTTP_VERSION* enums set below. */ + CINIT(HTTP_VERSION, LONG, 84), + + /* Specifically switch on or off the FTP engine's use of the EPSV command. By + default, that one will always be attempted before the more traditional + PASV command. */ + CINIT(FTP_USE_EPSV, LONG, 85), + + /* type of the file keeping your SSL-certificate ("DER", "PEM", "ENG") */ + CINIT(SSLCERTTYPE, STRINGPOINT, 86), + + /* name of the file keeping your private SSL-key */ + CINIT(SSLKEY, STRINGPOINT, 87), + + /* type of the file keeping your private SSL-key ("DER", "PEM", "ENG") */ + CINIT(SSLKEYTYPE, STRINGPOINT, 88), + + /* crypto engine for the SSL-sub system */ + CINIT(SSLENGINE, STRINGPOINT, 89), + + /* set the crypto engine for the SSL-sub system as default + the param has no meaning... + */ + CINIT(SSLENGINE_DEFAULT, LONG, 90), + + /* Non-zero value means to use the global dns cache */ + CINIT(DNS_USE_GLOBAL_CACHE, LONG, 91), /* DEPRECATED, do not use! */ + + /* DNS cache timeout */ + CINIT(DNS_CACHE_TIMEOUT, LONG, 92), + + /* send linked-list of pre-transfer QUOTE commands */ + CINIT(PREQUOTE, OBJECTPOINT, 93), + + /* set the debug function */ + CINIT(DEBUGFUNCTION, FUNCTIONPOINT, 94), + + /* set the data for the debug function */ + CINIT(DEBUGDATA, OBJECTPOINT, 95), + + /* mark this as start of a cookie session */ + CINIT(COOKIESESSION, LONG, 96), + + /* The CApath directory used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAPATH, STRINGPOINT, 97), + + /* Instruct libcurl to use a smaller receive buffer */ + CINIT(BUFFERSIZE, LONG, 98), + + /* Instruct libcurl to not use any signal/alarm handlers, even when using + timeouts. This option is useful for multi-threaded applications. + See libcurl-the-guide for more background information. */ + CINIT(NOSIGNAL, LONG, 99), + + /* Provide a CURLShare for mutexing non-ts data */ + CINIT(SHARE, OBJECTPOINT, 100), + + /* indicates type of proxy. accepted values are CURLPROXY_HTTP (default), + CURLPROXY_SOCKS4, CURLPROXY_SOCKS4A and CURLPROXY_SOCKS5. */ + CINIT(PROXYTYPE, LONG, 101), + + /* Set the Accept-Encoding string. Use this to tell a server you would like + the response to be compressed. Before 7.21.6, this was known as + CURLOPT_ENCODING */ + CINIT(ACCEPT_ENCODING, STRINGPOINT, 102), + + /* Set pointer to private data */ + CINIT(PRIVATE, OBJECTPOINT, 103), + + /* Set aliases for HTTP 200 in the HTTP Response header */ + CINIT(HTTP200ALIASES, OBJECTPOINT, 104), + + /* Continue to send authentication (user+password) when following locations, + even when hostname changed. This can potentially send off the name + and password to whatever host the server decides. */ + CINIT(UNRESTRICTED_AUTH, LONG, 105), + + /* Specifically switch on or off the FTP engine's use of the EPRT command ( + it also disables the LPRT attempt). By default, those ones will always be + attempted before the good old traditional PORT command. */ + CINIT(FTP_USE_EPRT, LONG, 106), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_USERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(HTTPAUTH, LONG, 107), + + /* Set the ssl context callback function, currently only for OpenSSL ssl_ctx + in second argument. The function must be matching the + curl_ssl_ctx_callback proto. */ + CINIT(SSL_CTX_FUNCTION, FUNCTIONPOINT, 108), + + /* Set the userdata for the ssl context callback function's third + argument */ + CINIT(SSL_CTX_DATA, OBJECTPOINT, 109), + + /* FTP Option that causes missing dirs to be created on the remote server. + In 7.19.4 we introduced the convenience enums for this option using the + CURLFTP_CREATE_DIR prefix. + */ + CINIT(FTP_CREATE_MISSING_DIRS, LONG, 110), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_PROXYUSERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(PROXYAUTH, LONG, 111), + + /* FTP option that changes the timeout, in seconds, associated with + getting a response. This is different from transfer timeout time and + essentially places a demand on the FTP server to acknowledge commands + in a timely manner. */ + CINIT(FTP_RESPONSE_TIMEOUT, LONG, 112), +#define CURLOPT_SERVER_RESPONSE_TIMEOUT CURLOPT_FTP_RESPONSE_TIMEOUT + + /* Set this option to one of the CURL_IPRESOLVE_* defines (see below) to + tell libcurl to resolve names to those IP versions only. This only has + affect on systems with support for more than one, i.e IPv4 _and_ IPv6. */ + CINIT(IPRESOLVE, LONG, 113), + + /* Set this option to limit the size of a file that will be downloaded from + an HTTP or FTP server. + + Note there is also _LARGE version which adds large file support for + platforms which have larger off_t sizes. See MAXFILESIZE_LARGE below. */ + CINIT(MAXFILESIZE, LONG, 114), + + /* See the comment for INFILESIZE above, but in short, specifies + * the size of the file being uploaded. -1 means unknown. + */ + CINIT(INFILESIZE_LARGE, OFF_T, 115), + + /* Sets the continuation offset. There is also a LONG version of this; + * look above for RESUME_FROM. + */ + CINIT(RESUME_FROM_LARGE, OFF_T, 116), + + /* Sets the maximum size of data that will be downloaded from + * an HTTP or FTP server. See MAXFILESIZE above for the LONG version. + */ + CINIT(MAXFILESIZE_LARGE, OFF_T, 117), + + /* Set this option to the file name of your .netrc file you want libcurl + to parse (using the CURLOPT_NETRC option). If not set, libcurl will do + a poor attempt to find the user's home directory and check for a .netrc + file in there. */ + CINIT(NETRC_FILE, STRINGPOINT, 118), + + /* Enable SSL/TLS for FTP, pick one of: + CURLUSESSL_TRY - try using SSL, proceed anyway otherwise + CURLUSESSL_CONTROL - SSL for the control connection or fail + CURLUSESSL_ALL - SSL for all communication or fail + */ + CINIT(USE_SSL, LONG, 119), + + /* The _LARGE version of the standard POSTFIELDSIZE option */ + CINIT(POSTFIELDSIZE_LARGE, OFF_T, 120), + + /* Enable/disable the TCP Nagle algorithm */ + CINIT(TCP_NODELAY, LONG, 121), + + /* 122 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 123 OBSOLETE. Gone in 7.16.0 */ + /* 124 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 125 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 126 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 127 OBSOLETE. Gone in 7.16.0 */ + /* 128 OBSOLETE. Gone in 7.16.0 */ + + /* When FTP over SSL/TLS is selected (with CURLOPT_USE_SSL), this option + can be used to change libcurl's default action which is to first try + "AUTH SSL" and then "AUTH TLS" in this order, and proceed when a OK + response has been received. + + Available parameters are: + CURLFTPAUTH_DEFAULT - let libcurl decide + CURLFTPAUTH_SSL - try "AUTH SSL" first, then TLS + CURLFTPAUTH_TLS - try "AUTH TLS" first, then SSL + */ + CINIT(FTPSSLAUTH, LONG, 129), + + CINIT(IOCTLFUNCTION, FUNCTIONPOINT, 130), + CINIT(IOCTLDATA, OBJECTPOINT, 131), + + /* 132 OBSOLETE. Gone in 7.16.0 */ + /* 133 OBSOLETE. Gone in 7.16.0 */ + + /* zero terminated string for pass on to the FTP server when asked for + "account" info */ + CINIT(FTP_ACCOUNT, STRINGPOINT, 134), + + /* feed cookie into cookie engine */ + CINIT(COOKIELIST, STRINGPOINT, 135), + + /* ignore Content-Length */ + CINIT(IGNORE_CONTENT_LENGTH, LONG, 136), + + /* Set to non-zero to skip the IP address received in a 227 PASV FTP server + response. Typically used for FTP-SSL purposes but is not restricted to + that. libcurl will then instead use the same IP address it used for the + control connection. */ + CINIT(FTP_SKIP_PASV_IP, LONG, 137), + + /* Select "file method" to use when doing FTP, see the curl_ftpmethod + above. */ + CINIT(FTP_FILEMETHOD, LONG, 138), + + /* Local port number to bind the socket to */ + CINIT(LOCALPORT, LONG, 139), + + /* Number of ports to try, including the first one set with LOCALPORT. + Thus, setting it to 1 will make no additional attempts but the first. + */ + CINIT(LOCALPORTRANGE, LONG, 140), + + /* no transfer, set up connection and let application use the socket by + extracting it with CURLINFO_LASTSOCKET */ + CINIT(CONNECT_ONLY, LONG, 141), + + /* Function that will be called to convert from the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_FROM_NETWORK_FUNCTION, FUNCTIONPOINT, 142), + + /* Function that will be called to convert to the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_TO_NETWORK_FUNCTION, FUNCTIONPOINT, 143), + + /* Function that will be called to convert from UTF8 + (instead of using the iconv calls in libcurl) + Note that this is used only for SSL certificate processing */ + CINIT(CONV_FROM_UTF8_FUNCTION, FUNCTIONPOINT, 144), + + /* if the connection proceeds too quickly then need to slow it down */ + /* limit-rate: maximum number of bytes per second to send or receive */ + CINIT(MAX_SEND_SPEED_LARGE, OFF_T, 145), + CINIT(MAX_RECV_SPEED_LARGE, OFF_T, 146), + + /* Pointer to command string to send if USER/PASS fails. */ + CINIT(FTP_ALTERNATIVE_TO_USER, STRINGPOINT, 147), + + /* callback function for setting socket options */ + CINIT(SOCKOPTFUNCTION, FUNCTIONPOINT, 148), + CINIT(SOCKOPTDATA, OBJECTPOINT, 149), + + /* set to 0 to disable session ID re-use for this transfer, default is + enabled (== 1) */ + CINIT(SSL_SESSIONID_CACHE, LONG, 150), + + /* allowed SSH authentication methods */ + CINIT(SSH_AUTH_TYPES, LONG, 151), + + /* Used by scp/sftp to do public/private key authentication */ + CINIT(SSH_PUBLIC_KEYFILE, STRINGPOINT, 152), + CINIT(SSH_PRIVATE_KEYFILE, STRINGPOINT, 153), + + /* Send CCC (Clear Command Channel) after authentication */ + CINIT(FTP_SSL_CCC, LONG, 154), + + /* Same as TIMEOUT and CONNECTTIMEOUT, but with ms resolution */ + CINIT(TIMEOUT_MS, LONG, 155), + CINIT(CONNECTTIMEOUT_MS, LONG, 156), + + /* set to zero to disable the libcurl's decoding and thus pass the raw body + data to the application even when it is encoded/compressed */ + CINIT(HTTP_TRANSFER_DECODING, LONG, 157), + CINIT(HTTP_CONTENT_DECODING, LONG, 158), + + /* Permission used when creating new files and directories on the remote + server for protocols that support it, SFTP/SCP/FILE */ + CINIT(NEW_FILE_PERMS, LONG, 159), + CINIT(NEW_DIRECTORY_PERMS, LONG, 160), + + /* Set the behaviour of POST when redirecting. Values must be set to one + of CURL_REDIR* defines below. This used to be called CURLOPT_POST301 */ + CINIT(POSTREDIR, LONG, 161), + + /* used by scp/sftp to verify the host's public key */ + CINIT(SSH_HOST_PUBLIC_KEY_MD5, STRINGPOINT, 162), + + /* Callback function for opening socket (instead of socket(2)). Optionally, + callback is able change the address or refuse to connect returning + CURL_SOCKET_BAD. The callback should have type + curl_opensocket_callback */ + CINIT(OPENSOCKETFUNCTION, FUNCTIONPOINT, 163), + CINIT(OPENSOCKETDATA, OBJECTPOINT, 164), + + /* POST volatile input fields. */ + CINIT(COPYPOSTFIELDS, OBJECTPOINT, 165), + + /* set transfer mode (;type=) when doing FTP via an HTTP proxy */ + CINIT(PROXY_TRANSFER_MODE, LONG, 166), + + /* Callback function for seeking in the input stream */ + CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167), + CINIT(SEEKDATA, OBJECTPOINT, 168), + + /* CRL file */ + CINIT(CRLFILE, STRINGPOINT, 169), + + /* Issuer certificate */ + CINIT(ISSUERCERT, STRINGPOINT, 170), + + /* (IPv6) Address scope */ + CINIT(ADDRESS_SCOPE, LONG, 171), + + /* Collect certificate chain info and allow it to get retrievable with + CURLINFO_CERTINFO after the transfer is complete. */ + CINIT(CERTINFO, LONG, 172), + + /* "name" and "pwd" to use when fetching. */ + CINIT(USERNAME, STRINGPOINT, 173), + CINIT(PASSWORD, STRINGPOINT, 174), + + /* "name" and "pwd" to use with Proxy when fetching. */ + CINIT(PROXYUSERNAME, STRINGPOINT, 175), + CINIT(PROXYPASSWORD, STRINGPOINT, 176), + + /* Comma separated list of hostnames defining no-proxy zones. These should + match both hostnames directly, and hostnames within a domain. For + example, local.com will match local.com and www.local.com, but NOT + notlocal.com or www.notlocal.com. For compatibility with other + implementations of this, .local.com will be considered to be the same as + local.com. A single * is the only valid wildcard, and effectively + disables the use of proxy. */ + CINIT(NOPROXY, STRINGPOINT, 177), + + /* block size for TFTP transfers */ + CINIT(TFTP_BLKSIZE, LONG, 178), + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_SERVICE, STRINGPOINT, 179), + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_NEC, LONG, 180), + + /* set the bitmask for the protocols that are allowed to be used for the + transfer, which thus helps the app which takes URLs from users or other + external inputs and want to restrict what protocol(s) to deal + with. Defaults to CURLPROTO_ALL. */ + CINIT(PROTOCOLS, LONG, 181), + + /* set the bitmask for the protocols that libcurl is allowed to follow to, + as a subset of the CURLOPT_PROTOCOLS ones. That means the protocol needs + to be set in both bitmasks to be allowed to get redirected to. Defaults + to all protocols except FILE and SCP. */ + CINIT(REDIR_PROTOCOLS, LONG, 182), + + /* set the SSH knownhost file name to use */ + CINIT(SSH_KNOWNHOSTS, STRINGPOINT, 183), + + /* set the SSH host key callback, must point to a curl_sshkeycallback + function */ + CINIT(SSH_KEYFUNCTION, FUNCTIONPOINT, 184), + + /* set the SSH host key callback custom pointer */ + CINIT(SSH_KEYDATA, OBJECTPOINT, 185), + + /* set the SMTP mail originator */ + CINIT(MAIL_FROM, STRINGPOINT, 186), + + /* set the list of SMTP mail receiver(s) */ + CINIT(MAIL_RCPT, OBJECTPOINT, 187), + + /* FTP: send PRET before PASV */ + CINIT(FTP_USE_PRET, LONG, 188), + + /* RTSP request method (OPTIONS, SETUP, PLAY, etc...) */ + CINIT(RTSP_REQUEST, LONG, 189), + + /* The RTSP session identifier */ + CINIT(RTSP_SESSION_ID, STRINGPOINT, 190), + + /* The RTSP stream URI */ + CINIT(RTSP_STREAM_URI, STRINGPOINT, 191), + + /* The Transport: header to use in RTSP requests */ + CINIT(RTSP_TRANSPORT, STRINGPOINT, 192), + + /* Manually initialize the client RTSP CSeq for this handle */ + CINIT(RTSP_CLIENT_CSEQ, LONG, 193), + + /* Manually initialize the server RTSP CSeq for this handle */ + CINIT(RTSP_SERVER_CSEQ, LONG, 194), + + /* The stream to pass to INTERLEAVEFUNCTION. */ + CINIT(INTERLEAVEDATA, OBJECTPOINT, 195), + + /* Let the application define a custom write method for RTP data */ + CINIT(INTERLEAVEFUNCTION, FUNCTIONPOINT, 196), + + /* Turn on wildcard matching */ + CINIT(WILDCARDMATCH, LONG, 197), + + /* Directory matching callback called before downloading of an + individual file (chunk) started */ + CINIT(CHUNK_BGN_FUNCTION, FUNCTIONPOINT, 198), + + /* Directory matching callback called after the file (chunk) + was downloaded, or skipped */ + CINIT(CHUNK_END_FUNCTION, FUNCTIONPOINT, 199), + + /* Change match (fnmatch-like) callback for wildcard matching */ + CINIT(FNMATCH_FUNCTION, FUNCTIONPOINT, 200), + + /* Let the application define custom chunk data pointer */ + CINIT(CHUNK_DATA, OBJECTPOINT, 201), + + /* FNMATCH_FUNCTION user pointer */ + CINIT(FNMATCH_DATA, OBJECTPOINT, 202), + + /* send linked-list of name:port:address sets */ + CINIT(RESOLVE, OBJECTPOINT, 203), + + /* Set a username for authenticated TLS */ + CINIT(TLSAUTH_USERNAME, STRINGPOINT, 204), + + /* Set a password for authenticated TLS */ + CINIT(TLSAUTH_PASSWORD, STRINGPOINT, 205), + + /* Set authentication type for authenticated TLS */ + CINIT(TLSAUTH_TYPE, STRINGPOINT, 206), + + /* Set to 1 to enable the "TE:" header in HTTP requests to ask for + compressed transfer-encoded responses. Set to 0 to disable the use of TE: + in outgoing requests. The current default is 0, but it might change in a + future libcurl release. + + libcurl will ask for the compressed methods it knows of, and if that + isn't any, it will not ask for transfer-encoding at all even if this + option is set to 1. + + */ + CINIT(TRANSFER_ENCODING, LONG, 207), + + /* Callback function for closing socket (instead of close(2)). The callback + should have type curl_closesocket_callback */ + CINIT(CLOSESOCKETFUNCTION, FUNCTIONPOINT, 208), + CINIT(CLOSESOCKETDATA, OBJECTPOINT, 209), + + /* allow GSSAPI credential delegation */ + CINIT(GSSAPI_DELEGATION, LONG, 210), + + /* Set the name servers to use for DNS resolution */ + CINIT(DNS_SERVERS, STRINGPOINT, 211), + + /* Time-out accept operations (currently for FTP only) after this amount + of miliseconds. */ + CINIT(ACCEPTTIMEOUT_MS, LONG, 212), + + /* Set TCP keepalive */ + CINIT(TCP_KEEPALIVE, LONG, 213), + + /* non-universal keepalive knobs (Linux, AIX, HP-UX, more) */ + CINIT(TCP_KEEPIDLE, LONG, 214), + CINIT(TCP_KEEPINTVL, LONG, 215), + + /* Enable/disable specific SSL features with a bitmask, see CURLSSLOPT_* */ + CINIT(SSL_OPTIONS, LONG, 216), + + /* Set the SMTP auth originator */ + CINIT(MAIL_AUTH, STRINGPOINT, 217), + + /* Enable/disable SASL initial response */ + CINIT(SASL_IR, LONG, 218), + + /* Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_xferinfo_callback + * prototype defines. (Deprecates CURLOPT_PROGRESSFUNCTION) */ + CINIT(XFERINFOFUNCTION, FUNCTIONPOINT, 219), + + /* The XOAUTH2 bearer token */ + CINIT(XOAUTH2_BEARER, STRINGPOINT, 220), + + /* Set the interface string to use as outgoing network + * interface for DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_INTERFACE, STRINGPOINT, 221), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP4, STRINGPOINT, 222), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP6, STRINGPOINT, 223), + + /* Set authentication options directly */ + CINIT(LOGIN_OPTIONS, STRINGPOINT, 224), + + /* Enable/disable TLS NPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_NPN, LONG, 225), + + /* Enable/disable TLS ALPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_ALPN, LONG, 226), + + /* Time to wait for a response to a HTTP request containing an + * Expect: 100-continue header before sending the data anyway. */ + CINIT(EXPECT_100_TIMEOUT_MS, LONG, 227), + + /* This points to a linked list of headers used for proxy requests only, + struct curl_slist kind */ + CINIT(PROXYHEADER, OBJECTPOINT, 228), + + /* Pass in a bitmask of "header options" */ + CINIT(HEADEROPT, LONG, 229), + + /* The public key in DER form used to validate the peer public key + this option is used only if SSL_VERIFYPEER is true */ + CINIT(PINNEDPUBLICKEY, STRINGPOINT, 230), + + /* Path to Unix domain socket */ + CINIT(UNIX_SOCKET_PATH, STRINGPOINT, 231), + + /* Set if we should verify the certificate status. */ + CINIT(SSL_VERIFYSTATUS, LONG, 232), + + /* Set if we should enable TLS false start. */ + CINIT(SSL_FALSESTART, LONG, 233), + + /* Do not squash dot-dot sequences */ + CINIT(PATH_AS_IS, LONG, 234), + + /* Proxy Service Name */ + CINIT(PROXY_SERVICE_NAME, STRINGPOINT, 235), + + /* Service Name */ + CINIT(SERVICE_NAME, STRINGPOINT, 236), + + /* Wait/don't wait for pipe/mutex to clarify */ + CINIT(PIPEWAIT, LONG, 237), + + /* Set the protocol used when curl is given a URL without a protocol */ + CINIT(DEFAULT_PROTOCOL, STRINGPOINT, 238), + + /* Set stream weight, 1 - 256 (default is 16) */ + CINIT(STREAM_WEIGHT, LONG, 239), + + /* Set stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS, OBJECTPOINT, 240), + + /* Set E-xclusive stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS_E, OBJECTPOINT, 241), + + CURLOPT_LASTENTRY /* the last unused */ +} CURLoption; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2011 */ + +/* This was added in version 7.19.1 */ +#define CURLOPT_POST301 CURLOPT_POSTREDIR + +/* These are scheduled to disappear by 2009 */ + +/* The following were added in 7.17.0 */ +#define CURLOPT_SSLKEYPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_FTPAPPEND CURLOPT_APPEND +#define CURLOPT_FTPLISTONLY CURLOPT_DIRLISTONLY +#define CURLOPT_FTP_SSL CURLOPT_USE_SSL + +/* The following were added earlier */ + +#define CURLOPT_SSLCERTPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_KRB4LEVEL CURLOPT_KRBLEVEL + +#else +/* This is set if CURL_NO_OLDIES is defined at compile-time */ +#undef CURLOPT_DNS_USE_GLOBAL_CACHE /* soon obsolete */ +#endif + + + /* Below here follows defines for the CURLOPT_IPRESOLVE option. If a host + name resolves addresses using more than one IP protocol version, this + option might be handy to force libcurl to use a specific IP version. */ +#define CURL_IPRESOLVE_WHATEVER 0 /* default, resolves addresses to all IP + versions that your system allows */ +#define CURL_IPRESOLVE_V4 1 /* resolve to IPv4 addresses */ +#define CURL_IPRESOLVE_V6 2 /* resolve to IPv6 addresses */ + + /* three convenient "aliases" that follow the name scheme better */ +#define CURLOPT_RTSPHEADER CURLOPT_HTTPHEADER + + /* These enums are for use with the CURLOPT_HTTP_VERSION option. */ +enum { + CURL_HTTP_VERSION_NONE, /* setting this means we don't care, and that we'd + like the library to choose the best possible + for us! */ + CURL_HTTP_VERSION_1_0, /* please use HTTP 1.0 in the request */ + CURL_HTTP_VERSION_1_1, /* please use HTTP 1.1 in the request */ + CURL_HTTP_VERSION_2_0, /* please use HTTP 2 in the request */ + CURL_HTTP_VERSION_2TLS, /* use version 2 for HTTPS, version 1.1 for HTTP */ + + CURL_HTTP_VERSION_LAST /* *ILLEGAL* http version */ +}; + +/* Convenience definition simple because the name of the version is HTTP/2 and + not 2.0. The 2_0 version of the enum name was set while the version was + still planned to be 2.0 and we stick to it for compatibility. */ +#define CURL_HTTP_VERSION_2 CURL_HTTP_VERSION_2_0 + +/* + * Public API enums for RTSP requests + */ +enum { + CURL_RTSPREQ_NONE, /* first in list */ + CURL_RTSPREQ_OPTIONS, + CURL_RTSPREQ_DESCRIBE, + CURL_RTSPREQ_ANNOUNCE, + CURL_RTSPREQ_SETUP, + CURL_RTSPREQ_PLAY, + CURL_RTSPREQ_PAUSE, + CURL_RTSPREQ_TEARDOWN, + CURL_RTSPREQ_GET_PARAMETER, + CURL_RTSPREQ_SET_PARAMETER, + CURL_RTSPREQ_RECORD, + CURL_RTSPREQ_RECEIVE, + CURL_RTSPREQ_LAST /* last in list */ +}; + + /* These enums are for use with the CURLOPT_NETRC option. */ +enum CURL_NETRC_OPTION { + CURL_NETRC_IGNORED, /* The .netrc will never be read. + * This is the default. */ + CURL_NETRC_OPTIONAL, /* A user:password in the URL will be preferred + * to one in the .netrc. */ + CURL_NETRC_REQUIRED, /* A user:password in the URL will be ignored. + * Unless one is set programmatically, the .netrc + * will be queried. */ + CURL_NETRC_LAST +}; + +enum { + CURL_SSLVERSION_DEFAULT, + CURL_SSLVERSION_TLSv1, /* TLS 1.x */ + CURL_SSLVERSION_SSLv2, + CURL_SSLVERSION_SSLv3, + CURL_SSLVERSION_TLSv1_0, + CURL_SSLVERSION_TLSv1_1, + CURL_SSLVERSION_TLSv1_2, + + CURL_SSLVERSION_LAST /* never use, keep last */ +}; + +enum CURL_TLSAUTH { + CURL_TLSAUTH_NONE, + CURL_TLSAUTH_SRP, + CURL_TLSAUTH_LAST /* never use, keep last */ +}; + +/* symbols to use with CURLOPT_POSTREDIR. + CURL_REDIR_POST_301, CURL_REDIR_POST_302 and CURL_REDIR_POST_303 + can be bitwise ORed so that CURL_REDIR_POST_301 | CURL_REDIR_POST_302 + | CURL_REDIR_POST_303 == CURL_REDIR_POST_ALL */ + +#define CURL_REDIR_GET_ALL 0 +#define CURL_REDIR_POST_301 1 +#define CURL_REDIR_POST_302 2 +#define CURL_REDIR_POST_303 4 +#define CURL_REDIR_POST_ALL \ + (CURL_REDIR_POST_301|CURL_REDIR_POST_302|CURL_REDIR_POST_303) + +typedef enum { + CURL_TIMECOND_NONE, + + CURL_TIMECOND_IFMODSINCE, + CURL_TIMECOND_IFUNMODSINCE, + CURL_TIMECOND_LASTMOD, + + CURL_TIMECOND_LAST +} curl_TimeCond; + + +/* curl_strequal() and curl_strnequal() are subject for removal in a future + libcurl, see lib/README.curlx for details */ +CURL_EXTERN int (curl_strequal)(const char *s1, const char *s2); +CURL_EXTERN int (curl_strnequal)(const char *s1, const char *s2, size_t n); + +/* name is uppercase CURLFORM_ */ +#ifdef CFINIT +#undef CFINIT +#endif + +#ifdef CURL_ISOCPP +#define CFINIT(name) CURLFORM_ ## name +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define CFINIT(name) CURLFORM_/**/name +#endif + +typedef enum { + CFINIT(NOTHING), /********* the first one is unused ************/ + + /* */ + CFINIT(COPYNAME), + CFINIT(PTRNAME), + CFINIT(NAMELENGTH), + CFINIT(COPYCONTENTS), + CFINIT(PTRCONTENTS), + CFINIT(CONTENTSLENGTH), + CFINIT(FILECONTENT), + CFINIT(ARRAY), + CFINIT(OBSOLETE), + CFINIT(FILE), + + CFINIT(BUFFER), + CFINIT(BUFFERPTR), + CFINIT(BUFFERLENGTH), + + CFINIT(CONTENTTYPE), + CFINIT(CONTENTHEADER), + CFINIT(FILENAME), + CFINIT(END), + CFINIT(OBSOLETE2), + + CFINIT(STREAM), + CFINIT(CONTENTLEN), /* added in 7.46.0, provide a curl_off_t length */ + + CURLFORM_LASTENTRY /* the last unused */ +} CURLformoption; + +#undef CFINIT /* done */ + +/* structure to be used as parameter for CURLFORM_ARRAY */ +struct curl_forms { + CURLformoption option; + const char *value; +}; + +/* use this for multipart formpost building */ +/* Returns code for curl_formadd() + * + * Returns: + * CURL_FORMADD_OK on success + * CURL_FORMADD_MEMORY if the FormInfo allocation fails + * CURL_FORMADD_OPTION_TWICE if one option is given twice for one Form + * CURL_FORMADD_NULL if a null pointer was given for a char + * CURL_FORMADD_MEMORY if the allocation of a FormInfo struct failed + * CURL_FORMADD_UNKNOWN_OPTION if an unknown option was used + * CURL_FORMADD_INCOMPLETE if the some FormInfo is not complete (or error) + * CURL_FORMADD_MEMORY if a curl_httppost struct cannot be allocated + * CURL_FORMADD_MEMORY if some allocation for string copying failed. + * CURL_FORMADD_ILLEGAL_ARRAY if an illegal option is used in an array + * + ***************************************************************************/ +typedef enum { + CURL_FORMADD_OK, /* first, no error */ + + CURL_FORMADD_MEMORY, + CURL_FORMADD_OPTION_TWICE, + CURL_FORMADD_NULL, + CURL_FORMADD_UNKNOWN_OPTION, + CURL_FORMADD_INCOMPLETE, + CURL_FORMADD_ILLEGAL_ARRAY, + CURL_FORMADD_DISABLED, /* libcurl was built with this disabled */ + + CURL_FORMADD_LAST /* last */ +} CURLFORMcode; + +/* + * NAME curl_formadd() + * + * DESCRIPTION + * + * Pretty advanced function for building multi-part formposts. Each invoke + * adds one part that together construct a full post. Then use + * CURLOPT_HTTPPOST to send it off to libcurl. + */ +CURL_EXTERN CURLFORMcode curl_formadd(struct curl_httppost **httppost, + struct curl_httppost **last_post, + ...); + +/* + * callback function for curl_formget() + * The void *arg pointer will be the one passed as second argument to + * curl_formget(). + * The character buffer passed to it must not be freed. + * Should return the buffer length passed to it as the argument "len" on + * success. + */ +typedef size_t (*curl_formget_callback)(void *arg, const char *buf, + size_t len); + +/* + * NAME curl_formget() + * + * DESCRIPTION + * + * Serialize a curl_httppost struct built with curl_formadd(). + * Accepts a void pointer as second argument which will be passed to + * the curl_formget_callback function. + * Returns 0 on success. + */ +CURL_EXTERN int curl_formget(struct curl_httppost *form, void *arg, + curl_formget_callback append); +/* + * NAME curl_formfree() + * + * DESCRIPTION + * + * Free a multipart formpost previously built with curl_formadd(). + */ +CURL_EXTERN void curl_formfree(struct curl_httppost *form); + +/* + * NAME curl_getenv() + * + * DESCRIPTION + * + * Returns a malloc()'ed string that MUST be curl_free()ed after usage is + * complete. DEPRECATED - see lib/README.curlx + */ +CURL_EXTERN char *curl_getenv(const char *variable); + +/* + * NAME curl_version() + * + * DESCRIPTION + * + * Returns a static ascii string of the libcurl version. + */ +CURL_EXTERN char *curl_version(void); + +/* + * NAME curl_easy_escape() + * + * DESCRIPTION + * + * Escapes URL strings (converts all letters consider illegal in URLs to their + * %XX versions). This function returns a new allocated string or NULL if an + * error occurred. + */ +CURL_EXTERN char *curl_easy_escape(CURL *handle, + const char *string, + int length); + +/* the previous version: */ +CURL_EXTERN char *curl_escape(const char *string, + int length); + + +/* + * NAME curl_easy_unescape() + * + * DESCRIPTION + * + * Unescapes URL encoding in strings (converts all %XX codes to their 8bit + * versions). This function returns a new allocated string or NULL if an error + * occurred. + * Conversion Note: On non-ASCII platforms the ASCII %XX codes are + * converted into the host encoding. + */ +CURL_EXTERN char *curl_easy_unescape(CURL *handle, + const char *string, + int length, + int *outlength); + +/* the previous version */ +CURL_EXTERN char *curl_unescape(const char *string, + int length); + +/* + * NAME curl_free() + * + * DESCRIPTION + * + * Provided for de-allocation in the same translation unit that did the + * allocation. Added in libcurl 7.10 + */ +CURL_EXTERN void curl_free(void *p); + +/* + * NAME curl_global_init() + * + * DESCRIPTION + * + * curl_global_init() should be invoked exactly once for each application that + * uses libcurl and before any call of other libcurl functions. + * + * This function is not thread-safe! + */ +CURL_EXTERN CURLcode curl_global_init(long flags); + +/* + * NAME curl_global_init_mem() + * + * DESCRIPTION + * + * curl_global_init() or curl_global_init_mem() should be invoked exactly once + * for each application that uses libcurl. This function can be used to + * initialize libcurl and set user defined memory management callback + * functions. Users can implement memory management routines to check for + * memory leaks, check for mis-use of the curl library etc. User registered + * callback routines with be invoked by this library instead of the system + * memory management routines like malloc, free etc. + */ +CURL_EXTERN CURLcode curl_global_init_mem(long flags, + curl_malloc_callback m, + curl_free_callback f, + curl_realloc_callback r, + curl_strdup_callback s, + curl_calloc_callback c); + +/* + * NAME curl_global_cleanup() + * + * DESCRIPTION + * + * curl_global_cleanup() should be invoked exactly once for each application + * that uses libcurl + */ +CURL_EXTERN void curl_global_cleanup(void); + +/* linked-list structure for the CURLOPT_QUOTE option (and other) */ +struct curl_slist { + char *data; + struct curl_slist *next; +}; + +/* + * NAME curl_slist_append() + * + * DESCRIPTION + * + * Appends a string to a linked list. If no list exists, it will be created + * first. Returns the new list, after appending. + */ +CURL_EXTERN struct curl_slist *curl_slist_append(struct curl_slist *, + const char *); + +/* + * NAME curl_slist_free_all() + * + * DESCRIPTION + * + * free a previously built curl_slist. + */ +CURL_EXTERN void curl_slist_free_all(struct curl_slist *); + +/* + * NAME curl_getdate() + * + * DESCRIPTION + * + * Returns the time, in seconds since 1 Jan 1970 of the time string given in + * the first argument. The time argument in the second parameter is unused + * and should be set to NULL. + */ +CURL_EXTERN time_t curl_getdate(const char *p, const time_t *unused); + +/* info about the certificate chain, only for OpenSSL builds. Asked + for with CURLOPT_CERTINFO / CURLINFO_CERTINFO */ +struct curl_certinfo { + int num_of_certs; /* number of certificates with information */ + struct curl_slist **certinfo; /* for each index in this array, there's a + linked list with textual information in the + format "name: value" */ +}; + +/* enum for the different supported SSL backends */ +typedef enum { + CURLSSLBACKEND_NONE = 0, + CURLSSLBACKEND_OPENSSL = 1, + CURLSSLBACKEND_GNUTLS = 2, + CURLSSLBACKEND_NSS = 3, + CURLSSLBACKEND_OBSOLETE4 = 4, /* Was QSOSSL. */ + CURLSSLBACKEND_GSKIT = 5, + CURLSSLBACKEND_POLARSSL = 6, + CURLSSLBACKEND_CYASSL = 7, + CURLSSLBACKEND_SCHANNEL = 8, + CURLSSLBACKEND_DARWINSSL = 9, + CURLSSLBACKEND_AXTLS = 10, + CURLSSLBACKEND_MBEDTLS = 11 +} curl_sslbackend; + +/* Information about the SSL library used and the respective internal SSL + handle, which can be used to obtain further information regarding the + connection. Asked for with CURLINFO_TLS_SESSION. */ +struct curl_tlssessioninfo { + curl_sslbackend backend; + void *internals; +}; + +#define CURLINFO_STRING 0x100000 +#define CURLINFO_LONG 0x200000 +#define CURLINFO_DOUBLE 0x300000 +#define CURLINFO_SLIST 0x400000 +#define CURLINFO_SOCKET 0x500000 +#define CURLINFO_MASK 0x0fffff +#define CURLINFO_TYPEMASK 0xf00000 + +typedef enum { + CURLINFO_NONE, /* first, never use this */ + CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1, + CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2, + CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3, + CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4, + CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5, + CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6, + CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7, + CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8, + CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9, + CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10, + CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11, + CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12, + CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13, + CURLINFO_FILETIME = CURLINFO_LONG + 14, + CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15, + CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16, + CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17, + CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18, + CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19, + CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20, + CURLINFO_PRIVATE = CURLINFO_STRING + 21, + CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22, + CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23, + CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24, + CURLINFO_OS_ERRNO = CURLINFO_LONG + 25, + CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26, + CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27, + CURLINFO_COOKIELIST = CURLINFO_SLIST + 28, + CURLINFO_LASTSOCKET = CURLINFO_LONG + 29, + CURLINFO_FTP_ENTRY_PATH = CURLINFO_STRING + 30, + CURLINFO_REDIRECT_URL = CURLINFO_STRING + 31, + CURLINFO_PRIMARY_IP = CURLINFO_STRING + 32, + CURLINFO_APPCONNECT_TIME = CURLINFO_DOUBLE + 33, + CURLINFO_CERTINFO = CURLINFO_SLIST + 34, + CURLINFO_CONDITION_UNMET = CURLINFO_LONG + 35, + CURLINFO_RTSP_SESSION_ID = CURLINFO_STRING + 36, + CURLINFO_RTSP_CLIENT_CSEQ = CURLINFO_LONG + 37, + CURLINFO_RTSP_SERVER_CSEQ = CURLINFO_LONG + 38, + CURLINFO_RTSP_CSEQ_RECV = CURLINFO_LONG + 39, + CURLINFO_PRIMARY_PORT = CURLINFO_LONG + 40, + CURLINFO_LOCAL_IP = CURLINFO_STRING + 41, + CURLINFO_LOCAL_PORT = CURLINFO_LONG + 42, + CURLINFO_TLS_SESSION = CURLINFO_SLIST + 43, + CURLINFO_ACTIVESOCKET = CURLINFO_SOCKET + 44, + /* Fill in new entries below here! */ + + CURLINFO_LASTONE = 44 +} CURLINFO; + +/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as + CURLINFO_HTTP_CODE */ +#define CURLINFO_HTTP_CODE CURLINFO_RESPONSE_CODE + +typedef enum { + CURLCLOSEPOLICY_NONE, /* first, never use this */ + + CURLCLOSEPOLICY_OLDEST, + CURLCLOSEPOLICY_LEAST_RECENTLY_USED, + CURLCLOSEPOLICY_LEAST_TRAFFIC, + CURLCLOSEPOLICY_SLOWEST, + CURLCLOSEPOLICY_CALLBACK, + + CURLCLOSEPOLICY_LAST /* last, never use this */ +} curl_closepolicy; + +#define CURL_GLOBAL_SSL (1<<0) +#define CURL_GLOBAL_WIN32 (1<<1) +#define CURL_GLOBAL_ALL (CURL_GLOBAL_SSL|CURL_GLOBAL_WIN32) +#define CURL_GLOBAL_NOTHING 0 +#define CURL_GLOBAL_DEFAULT CURL_GLOBAL_ALL +#define CURL_GLOBAL_ACK_EINTR (1<<2) + + +/***************************************************************************** + * Setup defines, protos etc for the sharing stuff. + */ + +/* Different data locks for a single share */ +typedef enum { + CURL_LOCK_DATA_NONE = 0, + /* CURL_LOCK_DATA_SHARE is used internally to say that + * the locking is just made to change the internal state of the share + * itself. + */ + CURL_LOCK_DATA_SHARE, + CURL_LOCK_DATA_COOKIE, + CURL_LOCK_DATA_DNS, + CURL_LOCK_DATA_SSL_SESSION, + CURL_LOCK_DATA_CONNECT, + CURL_LOCK_DATA_LAST +} curl_lock_data; + +/* Different lock access types */ +typedef enum { + CURL_LOCK_ACCESS_NONE = 0, /* unspecified action */ + CURL_LOCK_ACCESS_SHARED = 1, /* for read perhaps */ + CURL_LOCK_ACCESS_SINGLE = 2, /* for write perhaps */ + CURL_LOCK_ACCESS_LAST /* never use */ +} curl_lock_access; + +typedef void (*curl_lock_function)(CURL *handle, + curl_lock_data data, + curl_lock_access locktype, + void *userptr); +typedef void (*curl_unlock_function)(CURL *handle, + curl_lock_data data, + void *userptr); + +typedef void CURLSH; + +typedef enum { + CURLSHE_OK, /* all is fine */ + CURLSHE_BAD_OPTION, /* 1 */ + CURLSHE_IN_USE, /* 2 */ + CURLSHE_INVALID, /* 3 */ + CURLSHE_NOMEM, /* 4 out of memory */ + CURLSHE_NOT_BUILT_IN, /* 5 feature not present in lib */ + CURLSHE_LAST /* never use */ +} CURLSHcode; + +typedef enum { + CURLSHOPT_NONE, /* don't use */ + CURLSHOPT_SHARE, /* specify a data type to share */ + CURLSHOPT_UNSHARE, /* specify which data type to stop sharing */ + CURLSHOPT_LOCKFUNC, /* pass in a 'curl_lock_function' pointer */ + CURLSHOPT_UNLOCKFUNC, /* pass in a 'curl_unlock_function' pointer */ + CURLSHOPT_USERDATA, /* pass in a user data pointer used in the lock/unlock + callback functions */ + CURLSHOPT_LAST /* never use */ +} CURLSHoption; + +CURL_EXTERN CURLSH *curl_share_init(void); +CURL_EXTERN CURLSHcode curl_share_setopt(CURLSH *, CURLSHoption option, ...); +CURL_EXTERN CURLSHcode curl_share_cleanup(CURLSH *); + +/**************************************************************************** + * Structures for querying information about the curl library at runtime. + */ + +typedef enum { + CURLVERSION_FIRST, + CURLVERSION_SECOND, + CURLVERSION_THIRD, + CURLVERSION_FOURTH, + CURLVERSION_LAST /* never actually use this */ +} CURLversion; + +/* The 'CURLVERSION_NOW' is the symbolic name meant to be used by + basically all programs ever that want to get version information. It is + meant to be a built-in version number for what kind of struct the caller + expects. If the struct ever changes, we redefine the NOW to another enum + from above. */ +#define CURLVERSION_NOW CURLVERSION_FOURTH + +typedef struct { + CURLversion age; /* age of the returned struct */ + const char *version; /* LIBCURL_VERSION */ + unsigned int version_num; /* LIBCURL_VERSION_NUM */ + const char *host; /* OS/host/cpu/machine when configured */ + int features; /* bitmask, see defines below */ + const char *ssl_version; /* human readable string */ + long ssl_version_num; /* not used anymore, always 0 */ + const char *libz_version; /* human readable string */ + /* protocols is terminated by an entry with a NULL protoname */ + const char * const *protocols; + + /* The fields below this were added in CURLVERSION_SECOND */ + const char *ares; + int ares_num; + + /* This field was added in CURLVERSION_THIRD */ + const char *libidn; + + /* These field were added in CURLVERSION_FOURTH */ + + /* Same as '_libiconv_version' if built with HAVE_ICONV */ + int iconv_ver_num; + + const char *libssh_version; /* human readable string */ + +} curl_version_info_data; + +#define CURL_VERSION_IPV6 (1<<0) /* IPv6-enabled */ +#define CURL_VERSION_KERBEROS4 (1<<1) /* Kerberos V4 auth is supported + (deprecated) */ +#define CURL_VERSION_SSL (1<<2) /* SSL options are present */ +#define CURL_VERSION_LIBZ (1<<3) /* libz features are present */ +#define CURL_VERSION_NTLM (1<<4) /* NTLM auth is supported */ +#define CURL_VERSION_GSSNEGOTIATE (1<<5) /* Negotiate auth is supported + (deprecated) */ +#define CURL_VERSION_DEBUG (1<<6) /* Built with debug capabilities */ +#define CURL_VERSION_ASYNCHDNS (1<<7) /* Asynchronous DNS resolves */ +#define CURL_VERSION_SPNEGO (1<<8) /* SPNEGO auth is supported */ +#define CURL_VERSION_LARGEFILE (1<<9) /* Supports files larger than 2GB */ +#define CURL_VERSION_IDN (1<<10) /* Internationized Domain Names are + supported */ +#define CURL_VERSION_SSPI (1<<11) /* Built against Windows SSPI */ +#define CURL_VERSION_CONV (1<<12) /* Character conversions supported */ +#define CURL_VERSION_CURLDEBUG (1<<13) /* Debug memory tracking supported */ +#define CURL_VERSION_TLSAUTH_SRP (1<<14) /* TLS-SRP auth is supported */ +#define CURL_VERSION_NTLM_WB (1<<15) /* NTLM delegation to winbind helper + is suported */ +#define CURL_VERSION_HTTP2 (1<<16) /* HTTP2 support built-in */ +#define CURL_VERSION_GSSAPI (1<<17) /* Built against a GSS-API library */ +#define CURL_VERSION_KERBEROS5 (1<<18) /* Kerberos V5 auth is supported */ +#define CURL_VERSION_UNIX_SOCKETS (1<<19) /* Unix domain sockets support */ +#define CURL_VERSION_PSL (1<<20) /* Mozilla's Public Suffix List, used + for cookie domain verification */ + + /* + * NAME curl_version_info() + * + * DESCRIPTION + * + * This function returns a pointer to a static copy of the version info + * struct. See above. + */ +CURL_EXTERN curl_version_info_data *curl_version_info(CURLversion); + +/* + * NAME curl_easy_strerror() + * + * DESCRIPTION + * + * The curl_easy_strerror function may be used to turn a CURLcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_easy_strerror(CURLcode); + +/* + * NAME curl_share_strerror() + * + * DESCRIPTION + * + * The curl_share_strerror function may be used to turn a CURLSHcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_share_strerror(CURLSHcode); + +/* + * NAME curl_easy_pause() + * + * DESCRIPTION + * + * The curl_easy_pause function pauses or unpauses transfers. Select the new + * state by setting the bitmask, use the convenience defines below. + * + */ +CURL_EXTERN CURLcode curl_easy_pause(CURL *handle, int bitmask); + +#define CURLPAUSE_RECV (1<<0) +#define CURLPAUSE_RECV_CONT (0) + +#define CURLPAUSE_SEND (1<<2) +#define CURLPAUSE_SEND_CONT (0) + +#define CURLPAUSE_ALL (CURLPAUSE_RECV|CURLPAUSE_SEND) +#define CURLPAUSE_CONT (CURLPAUSE_RECV_CONT|CURLPAUSE_SEND_CONT) + +#ifdef __cplusplus +} +#endif + +/* unfortunately, the easy.h and multi.h include files need options and info + stuff before they can be included! */ +#include "easy.h" /* nothing in curl is fun without the easy stuff */ +#include "multi.h" + +/* the typechecker doesn't work in C++ (yet) */ +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && \ + !defined(__cplusplus) && !defined(CURL_DISABLE_TYPECHECK) +#include "typecheck-gcc.h" +#else +#if defined(__STDC__) && (__STDC__ >= 1) +/* This preprocessor magic that replaces a call with the exact same call is + only done to make sure application authors pass exactly three arguments + to these functions. */ +#define curl_easy_setopt(handle,opt,param) curl_easy_setopt(handle,opt,param) +#define curl_easy_getinfo(handle,info,arg) curl_easy_getinfo(handle,info,arg) +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) +#endif /* __STDC__ >= 1 */ +#endif /* gcc >= 4.3 && !__cplusplus */ + +#endif /* __CURL_CURL_H */ diff --git a/deps/libcurl/include/curl/curlbuild.h b/deps/libcurl/include/curl/curlbuild.h new file mode 100644 index 0000000000000000000000000000000000000000..bdca52b5355feec5f80f279bea6c5799bcf1041e --- /dev/null +++ b/deps/libcurl/include/curl/curlbuild.h @@ -0,0 +1,198 @@ +/* include/curl/curlbuild.h. Generated from curlbuild.h.in by configure. */ +#ifndef __CURL_CURLBUILD_H +#define __CURL_CURLBUILD_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* ================================================================ */ +/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */ +/* ================================================================ */ + +/* + * NOTE 1: + * ------- + * + * Nothing in this file is intended to be modified or adjusted by the + * curl library user nor by the curl library builder. + * + * If you think that something actually needs to be changed, adjusted + * or fixed in this file, then, report it on the libcurl development + * mailing list: http://cool.haxx.se/mailman/listinfo/curl-library/ + * + * This header file shall only export symbols which are 'curl' or 'CURL' + * prefixed, otherwise public name space would be polluted. + * + * NOTE 2: + * ------- + * + * Right now you might be staring at file include/curl/curlbuild.h.in or + * at file include/curl/curlbuild.h, this is due to the following reason: + * + * On systems capable of running the configure script, the configure process + * will overwrite the distributed include/curl/curlbuild.h file with one that + * is suitable and specific to the library being configured and built, which + * is generated from the include/curl/curlbuild.h.in template file. + * + */ + +/* ================================================================ */ +/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */ +/* ================================================================ */ + +#ifdef CURL_SIZEOF_LONG +#error "CURL_SIZEOF_LONG shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SIZEOF_LONG_already_defined +#endif + +#ifdef CURL_TYPEOF_CURL_SOCKLEN_T +#error "CURL_TYPEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_already_defined +#endif + +#ifdef CURL_SIZEOF_CURL_SOCKLEN_T +#error "CURL_SIZEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_already_defined +#endif + +#ifdef CURL_TYPEOF_CURL_OFF_T +#error "CURL_TYPEOF_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_FORMAT_CURL_OFF_T +#error "CURL_FORMAT_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_FORMAT_CURL_OFF_TU +#error "CURL_FORMAT_CURL_OFF_TU shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_already_defined +#endif + +#ifdef CURL_FORMAT_OFF_T +#error "CURL_FORMAT_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_FORMAT_OFF_T_already_defined +#endif + +#ifdef CURL_SIZEOF_CURL_OFF_T +#error "CURL_SIZEOF_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_SUFFIX_CURL_OFF_T +#error "CURL_SUFFIX_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_SUFFIX_CURL_OFF_TU +#error "CURL_SUFFIX_CURL_OFF_TU shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_already_defined +#endif + +/* ================================================================ */ +/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */ +/* ================================================================ */ + +/* Configure process defines this to 1 when it finds out that system */ +/* header file ws2tcpip.h must be included by the external interface. */ +/* #undef CURL_PULL_WS2TCPIP_H */ +#ifdef CURL_PULL_WS2TCPIP_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/types.h must be included by the external interface. */ +#define CURL_PULL_SYS_TYPES_H 1 +#ifdef CURL_PULL_SYS_TYPES_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file stdint.h must be included by the external interface. */ +/* #undef CURL_PULL_STDINT_H */ +#ifdef CURL_PULL_STDINT_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file inttypes.h must be included by the external interface. */ +/* #undef CURL_PULL_INTTYPES_H */ +#ifdef CURL_PULL_INTTYPES_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/socket.h must be included by the external interface. */ +#define CURL_PULL_SYS_SOCKET_H 1 +#ifdef CURL_PULL_SYS_SOCKET_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/poll.h must be included by the external interface. */ +/* #undef CURL_PULL_SYS_POLL_H */ +#ifdef CURL_PULL_SYS_POLL_H +# include +#endif + +/* The size of `long', as computed by sizeof. */ +#define CURL_SIZEOF_LONG 8 + +/* Integral data type used for curl_socklen_t. */ +#define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t + +/* The size of `curl_socklen_t', as computed by sizeof. */ +#define CURL_SIZEOF_CURL_SOCKLEN_T 4 + +/* Data type definition of curl_socklen_t. */ +typedef CURL_TYPEOF_CURL_SOCKLEN_T curl_socklen_t; + +/* Signed integral data type used for curl_off_t. */ +#define CURL_TYPEOF_CURL_OFF_T long + +/* Data type definition of curl_off_t. */ +typedef CURL_TYPEOF_CURL_OFF_T curl_off_t; + +/* curl_off_t formatting string directive without "%" conversion specifier. */ +#define CURL_FORMAT_CURL_OFF_T "ld" + +/* unsigned curl_off_t formatting string without "%" conversion specifier. */ +#define CURL_FORMAT_CURL_OFF_TU "lu" + +/* curl_off_t formatting string directive with "%" conversion specifier. */ +#define CURL_FORMAT_OFF_T "%ld" + +/* The size of `curl_off_t', as computed by sizeof. */ +#define CURL_SIZEOF_CURL_OFF_T 8 + +/* curl_off_t constant suffix. */ +#define CURL_SUFFIX_CURL_OFF_T L + +/* unsigned curl_off_t constant suffix. */ +#define CURL_SUFFIX_CURL_OFF_TU UL + +#endif /* __CURL_CURLBUILD_H */ diff --git a/deps/libcurl/include/curl/curlrules.h b/deps/libcurl/include/curl/curlrules.h new file mode 100644 index 0000000000000000000000000000000000000000..7c2ede35b63a7540acd042ce5c2b27a70c982d02 --- /dev/null +++ b/deps/libcurl/include/curl/curlrules.h @@ -0,0 +1,262 @@ +#ifndef __CURL_CURLRULES_H +#define __CURL_CURLRULES_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* ================================================================ */ +/* COMPILE TIME SANITY CHECKS */ +/* ================================================================ */ + +/* + * NOTE 1: + * ------- + * + * All checks done in this file are intentionally placed in a public + * header file which is pulled by curl/curl.h when an application is + * being built using an already built libcurl library. Additionally + * this file is also included and used when building the library. + * + * If compilation fails on this file it is certainly sure that the + * problem is elsewhere. It could be a problem in the curlbuild.h + * header file, or simply that you are using different compilation + * settings than those used to build the library. + * + * Nothing in this file is intended to be modified or adjusted by the + * curl library user nor by the curl library builder. + * + * Do not deactivate any check, these are done to make sure that the + * library is properly built and used. + * + * You can find further help on the libcurl development mailing list: + * http://cool.haxx.se/mailman/listinfo/curl-library/ + * + * NOTE 2 + * ------ + * + * Some of the following compile time checks are based on the fact + * that the dimension of a constant array can not be a negative one. + * In this way if the compile time verification fails, the compilation + * will fail issuing an error. The error description wording is compiler + * dependent but it will be quite similar to one of the following: + * + * "negative subscript or subscript is too large" + * "array must have at least one element" + * "-1 is an illegal array size" + * "size of array is negative" + * + * If you are building an application which tries to use an already + * built libcurl library and you are getting this kind of errors on + * this file, it is a clear indication that there is a mismatch between + * how the library was built and how you are trying to use it for your + * application. Your already compiled or binary library provider is the + * only one who can give you the details you need to properly use it. + */ + +/* + * Verify that some macros are actually defined. + */ + +#ifndef CURL_SIZEOF_LONG +# error "CURL_SIZEOF_LONG definition is missing!" + Error Compilation_aborted_CURL_SIZEOF_LONG_is_missing +#endif + +#ifndef CURL_TYPEOF_CURL_SOCKLEN_T +# error "CURL_TYPEOF_CURL_SOCKLEN_T definition is missing!" + Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_is_missing +#endif + +#ifndef CURL_SIZEOF_CURL_SOCKLEN_T +# error "CURL_SIZEOF_CURL_SOCKLEN_T definition is missing!" + Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_is_missing +#endif + +#ifndef CURL_TYPEOF_CURL_OFF_T +# error "CURL_TYPEOF_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_FORMAT_CURL_OFF_T +# error "CURL_FORMAT_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_FORMAT_CURL_OFF_TU +# error "CURL_FORMAT_CURL_OFF_TU definition is missing!" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_is_missing +#endif + +#ifndef CURL_FORMAT_OFF_T +# error "CURL_FORMAT_OFF_T definition is missing!" + Error Compilation_aborted_CURL_FORMAT_OFF_T_is_missing +#endif + +#ifndef CURL_SIZEOF_CURL_OFF_T +# error "CURL_SIZEOF_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_SUFFIX_CURL_OFF_T +# error "CURL_SUFFIX_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_SUFFIX_CURL_OFF_TU +# error "CURL_SUFFIX_CURL_OFF_TU definition is missing!" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_is_missing +#endif + +/* + * Macros private to this header file. + */ + +#define CurlchkszEQ(t, s) sizeof(t) == s ? 1 : -1 + +#define CurlchkszGE(t1, t2) sizeof(t1) >= sizeof(t2) ? 1 : -1 + +/* + * Verify that the size previously defined and expected for long + * is the same as the one reported by sizeof() at compile time. + */ + +typedef char + __curl_rule_01__ + [CurlchkszEQ(long, CURL_SIZEOF_LONG)]; + +/* + * Verify that the size previously defined and expected for + * curl_off_t is actually the the same as the one reported + * by sizeof() at compile time. + */ + +typedef char + __curl_rule_02__ + [CurlchkszEQ(curl_off_t, CURL_SIZEOF_CURL_OFF_T)]; + +/* + * Verify at compile time that the size of curl_off_t as reported + * by sizeof() is greater or equal than the one reported for long + * for the current compilation. + */ + +typedef char + __curl_rule_03__ + [CurlchkszGE(curl_off_t, long)]; + +/* + * Verify that the size previously defined and expected for + * curl_socklen_t is actually the the same as the one reported + * by sizeof() at compile time. + */ + +typedef char + __curl_rule_04__ + [CurlchkszEQ(curl_socklen_t, CURL_SIZEOF_CURL_SOCKLEN_T)]; + +/* + * Verify at compile time that the size of curl_socklen_t as reported + * by sizeof() is greater or equal than the one reported for int for + * the current compilation. + */ + +typedef char + __curl_rule_05__ + [CurlchkszGE(curl_socklen_t, int)]; + +/* ================================================================ */ +/* EXTERNALLY AND INTERNALLY VISIBLE DEFINITIONS */ +/* ================================================================ */ + +/* + * CURL_ISOCPP and CURL_OFF_T_C definitions are done here in order to allow + * these to be visible and exported by the external libcurl interface API, + * while also making them visible to the library internals, simply including + * curl_setup.h, without actually needing to include curl.h internally. + * If some day this section would grow big enough, all this should be moved + * to its own header file. + */ + +/* + * Figure out if we can use the ## preprocessor operator, which is supported + * by ISO/ANSI C and C++. Some compilers support it without setting __STDC__ + * or __cplusplus so we need to carefully check for them too. + */ + +#if defined(__STDC__) || defined(_MSC_VER) || defined(__cplusplus) || \ + defined(__HP_aCC) || defined(__BORLANDC__) || defined(__LCC__) || \ + defined(__POCC__) || defined(__SALFORDC__) || defined(__HIGHC__) || \ + defined(__ILEC400__) + /* This compiler is believed to have an ISO compatible preprocessor */ +#define CURL_ISOCPP +#else + /* This compiler is believed NOT to have an ISO compatible preprocessor */ +#undef CURL_ISOCPP +#endif + +/* + * Macros for minimum-width signed and unsigned curl_off_t integer constants. + */ + +#if defined(__BORLANDC__) && (__BORLANDC__ == 0x0551) +# define __CURL_OFF_T_C_HLPR2(x) x +# define __CURL_OFF_T_C_HLPR1(x) __CURL_OFF_T_C_HLPR2(x) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_TU) +#else +# ifdef CURL_ISOCPP +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val ## Suffix +# else +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val/**/Suffix +# endif +# define __CURL_OFF_T_C_HLPR1(Val,Suffix) __CURL_OFF_T_C_HLPR2(Val,Suffix) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_TU) +#endif + +/* + * Get rid of macros private to this header file. + */ + +#undef CurlchkszEQ +#undef CurlchkszGE + +/* + * Get rid of macros not intended to exist beyond this point. + */ + +#undef CURL_PULL_WS2TCPIP_H +#undef CURL_PULL_SYS_TYPES_H +#undef CURL_PULL_SYS_SOCKET_H +#undef CURL_PULL_SYS_POLL_H +#undef CURL_PULL_STDINT_H +#undef CURL_PULL_INTTYPES_H + +#undef CURL_TYPEOF_CURL_SOCKLEN_T +#undef CURL_TYPEOF_CURL_OFF_T + +#ifdef CURL_NO_OLDIES +#undef CURL_FORMAT_OFF_T /* not required since 7.19.0 - obsoleted in 7.20.0 */ +#endif + +#endif /* __CURL_CURLRULES_H */ diff --git a/deps/libcurl/include/curl/curlver.h b/deps/libcurl/include/curl/curlver.h new file mode 100644 index 0000000000000000000000000000000000000000..17906764c898e69ce28204bda7d74573f4c365ee --- /dev/null +++ b/deps/libcurl/include/curl/curlver.h @@ -0,0 +1,77 @@ +#ifndef __CURL_CURLVER_H +#define __CURL_CURLVER_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* This header file contains nothing but libcurl version info, generated by + a script at release-time. This was made its own header file in 7.11.2 */ + +/* This is the global package copyright */ +#define LIBCURL_COPYRIGHT "1996 - 2015 Daniel Stenberg, ." + +/* This is the version number of the libcurl package from which this header + file origins: */ +#define LIBCURL_VERSION "7.47.0" + +/* The numeric version number is also available "in parts" by using these + defines: */ +#define LIBCURL_VERSION_MAJOR 7 +#define LIBCURL_VERSION_MINOR 47 +#define LIBCURL_VERSION_PATCH 0 + +/* This is the numeric version of the libcurl version number, meant for easier + parsing and comparions by programs. The LIBCURL_VERSION_NUM define will + always follow this syntax: + + 0xXXYYZZ + + Where XX, YY and ZZ are the main version, release and patch numbers in + hexadecimal (using 8 bits each). All three numbers are always represented + using two digits. 1.2 would appear as "0x010200" while version 9.11.7 + appears as "0x090b07". + + This 6-digit (24 bits) hexadecimal number does not show pre-release number, + and it is always a greater number in a more recent release. It makes + comparisons with greater than and less than work. + + Note: This define is the full hex number and _does not_ use the + CURL_VERSION_BITS() macro since curl's own configure script greps for it + and needs it to contain the full number. +*/ +#define LIBCURL_VERSION_NUM 0x072f00 + +/* + * This is the date and time when the full source package was created. The + * timestamp is not stored in git, as the timestamp is properly set in the + * tarballs by the maketgz script. + * + * The format of the date should follow this template: + * + * "Mon Feb 12 11:35:33 UTC 2007" + */ +#define LIBCURL_TIMESTAMP "Wed Jan 27 07:32:44 UTC 2016" + +#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) +#define CURL_AT_LEAST_VERSION(x,y,z) \ + (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) + +#endif /* __CURL_CURLVER_H */ diff --git a/deps/libcurl/include/curl/easy.h b/deps/libcurl/include/curl/easy.h new file mode 100644 index 0000000000000000000000000000000000000000..c1e3e76096e3925821da279a2f2eae67ae61e0e8 --- /dev/null +++ b/deps/libcurl/include/curl/easy.h @@ -0,0 +1,102 @@ +#ifndef __CURL_EASY_H +#define __CURL_EASY_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2008, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN CURL *curl_easy_init(void); +CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); +CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); +CURL_EXTERN void curl_easy_cleanup(CURL *curl); + +/* + * NAME curl_easy_getinfo() + * + * DESCRIPTION + * + * Request internal information from the curl session with this function. The + * third argument MUST be a pointer to a long, a pointer to a char * or a + * pointer to a double (as the documentation describes elsewhere). The data + * pointed to will be filled in accordingly and can be relied upon only if the + * function returns CURLE_OK. This function is intended to get used *AFTER* a + * performed transfer, all results from this function are undefined until the + * transfer is completed. + */ +CURL_EXTERN CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...); + + +/* + * NAME curl_easy_duphandle() + * + * DESCRIPTION + * + * Creates a new curl session handle with the same options set for the handle + * passed in. Duplicating a handle could only be a matter of cloning data and + * options, internal state info and things like persistent connections cannot + * be transferred. It is useful in multithreaded applications when you can run + * curl_easy_duphandle() for each new thread to avoid a series of identical + * curl_easy_setopt() invokes in every thread. + */ +CURL_EXTERN CURL* curl_easy_duphandle(CURL *curl); + +/* + * NAME curl_easy_reset() + * + * DESCRIPTION + * + * Re-initializes a CURL handle to the default values. This puts back the + * handle to the same state as it was in when it was just created. + * + * It does keep: live connections, the Session ID cache, the DNS cache and the + * cookies. + */ +CURL_EXTERN void curl_easy_reset(CURL *curl); + +/* + * NAME curl_easy_recv() + * + * DESCRIPTION + * + * Receives data from the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen, + size_t *n); + +/* + * NAME curl_easy_send() + * + * DESCRIPTION + * + * Sends data over the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer, + size_t buflen, size_t *n); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deps/libcurl/include/curl/mprintf.h b/deps/libcurl/include/curl/mprintf.h new file mode 100644 index 0000000000000000000000000000000000000000..c6b0d7679a6cf970a7960af9f47a7485729598dc --- /dev/null +++ b/deps/libcurl/include/curl/mprintf.h @@ -0,0 +1,74 @@ +#ifndef __CURL_MPRINTF_H +#define __CURL_MPRINTF_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include +#include /* needed for FILE */ + +#include "curl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN int curl_mprintf(const char *format, ...); +CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...); +CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...); +CURL_EXTERN int curl_msnprintf(char *buffer, size_t maxlength, + const char *format, ...); +CURL_EXTERN int curl_mvprintf(const char *format, va_list args); +CURL_EXTERN int curl_mvfprintf(FILE *fd, const char *format, va_list args); +CURL_EXTERN int curl_mvsprintf(char *buffer, const char *format, va_list args); +CURL_EXTERN int curl_mvsnprintf(char *buffer, size_t maxlength, + const char *format, va_list args); +CURL_EXTERN char *curl_maprintf(const char *format, ...); +CURL_EXTERN char *curl_mvaprintf(const char *format, va_list args); + +#ifdef _MPRINTF_REPLACE +# undef printf +# undef fprintf +# undef sprintf +# undef vsprintf +# undef snprintf +# undef vprintf +# undef vfprintf +# undef vsnprintf +# undef aprintf +# undef vaprintf +# define printf curl_mprintf +# define fprintf curl_mfprintf +# define sprintf curl_msprintf +# define vsprintf curl_mvsprintf +# define snprintf curl_msnprintf +# define vprintf curl_mvprintf +# define vfprintf curl_mvfprintf +# define vsnprintf curl_mvsnprintf +# define aprintf curl_maprintf +# define vaprintf curl_mvaprintf +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __CURL_MPRINTF_H */ diff --git a/deps/libcurl/include/curl/multi.h b/deps/libcurl/include/curl/multi.h new file mode 100644 index 0000000000000000000000000000000000000000..36e2e940ecd440c0a14aadc0dc3eba1d5d53f186 --- /dev/null +++ b/deps/libcurl/include/curl/multi.h @@ -0,0 +1,435 @@ +#ifndef __CURL_MULTI_H +#define __CURL_MULTI_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +/* + This is an "external" header file. Don't give away any internals here! + + GOALS + + o Enable a "pull" interface. The application that uses libcurl decides where + and when to ask libcurl to get/send data. + + o Enable multiple simultaneous transfers in the same thread without making it + complicated for the application. + + o Enable the application to select() on its own file descriptors and curl's + file descriptors simultaneous easily. + +*/ + +/* + * This header file should not really need to include "curl.h" since curl.h + * itself includes this file and we expect user applications to do #include + * without the need for especially including multi.h. + * + * For some reason we added this include here at one point, and rather than to + * break existing (wrongly written) libcurl applications, we leave it as-is + * but with this warning attached. + */ +#include "curl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void CURLM; + +typedef enum { + CURLM_CALL_MULTI_PERFORM = -1, /* please call curl_multi_perform() or + curl_multi_socket*() soon */ + CURLM_OK, + CURLM_BAD_HANDLE, /* the passed-in handle is not a valid CURLM handle */ + CURLM_BAD_EASY_HANDLE, /* an easy handle was not good/valid */ + CURLM_OUT_OF_MEMORY, /* if you ever get this, you're in deep sh*t */ + CURLM_INTERNAL_ERROR, /* this is a libcurl bug */ + CURLM_BAD_SOCKET, /* the passed in socket argument did not match */ + CURLM_UNKNOWN_OPTION, /* curl_multi_setopt() with unsupported option */ + CURLM_ADDED_ALREADY, /* an easy handle already added to a multi handle was + attempted to get added - again */ + CURLM_LAST +} CURLMcode; + +/* just to make code nicer when using curl_multi_socket() you can now check + for CURLM_CALL_MULTI_SOCKET too in the same style it works for + curl_multi_perform() and CURLM_CALL_MULTI_PERFORM */ +#define CURLM_CALL_MULTI_SOCKET CURLM_CALL_MULTI_PERFORM + +/* bitmask bits for CURLMOPT_PIPELINING */ +#define CURLPIPE_NOTHING 0L +#define CURLPIPE_HTTP1 1L +#define CURLPIPE_MULTIPLEX 2L + +typedef enum { + CURLMSG_NONE, /* first, not used */ + CURLMSG_DONE, /* This easy handle has completed. 'result' contains + the CURLcode of the transfer */ + CURLMSG_LAST /* last, not used */ +} CURLMSG; + +struct CURLMsg { + CURLMSG msg; /* what this message means */ + CURL *easy_handle; /* the handle it concerns */ + union { + void *whatever; /* message-specific data */ + CURLcode result; /* return code for transfer */ + } data; +}; +typedef struct CURLMsg CURLMsg; + +/* Based on poll(2) structure and values. + * We don't use pollfd and POLL* constants explicitly + * to cover platforms without poll(). */ +#define CURL_WAIT_POLLIN 0x0001 +#define CURL_WAIT_POLLPRI 0x0002 +#define CURL_WAIT_POLLOUT 0x0004 + +struct curl_waitfd { + curl_socket_t fd; + short events; + short revents; /* not supported yet */ +}; + +/* + * Name: curl_multi_init() + * + * Desc: inititalize multi-style curl usage + * + * Returns: a new CURLM handle to use in all 'curl_multi' functions. + */ +CURL_EXTERN CURLM *curl_multi_init(void); + +/* + * Name: curl_multi_add_handle() + * + * Desc: add a standard curl handle to the multi stack + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_add_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_remove_handle() + * + * Desc: removes a curl handle from the multi stack again + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_remove_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_fdset() + * + * Desc: Ask curl for its fd_set sets. The app can use these to select() or + * poll() on. We want curl_multi_perform() called as soon as one of + * them are ready. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_fdset(CURLM *multi_handle, + fd_set *read_fd_set, + fd_set *write_fd_set, + fd_set *exc_fd_set, + int *max_fd); + +/* + * Name: curl_multi_wait() + * + * Desc: Poll on all fds within a CURLM set as well as any + * additional fds passed to the function. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_wait(CURLM *multi_handle, + struct curl_waitfd extra_fds[], + unsigned int extra_nfds, + int timeout_ms, + int *ret); + + /* + * Name: curl_multi_perform() + * + * Desc: When the app thinks there's data available for curl it calls this + * function to read/write whatever there is right now. This returns + * as soon as the reads and writes are done. This function does not + * require that there actually is data available for reading or that + * data can be written, it can be called just in case. It returns + * the number of handles that still transfer data in the second + * argument's integer-pointer. + * + * Returns: CURLMcode type, general multi error code. *NOTE* that this only + * returns errors etc regarding the whole multi stack. There might + * still have occurred problems on invidual transfers even when this + * returns OK. + */ +CURL_EXTERN CURLMcode curl_multi_perform(CURLM *multi_handle, + int *running_handles); + + /* + * Name: curl_multi_cleanup() + * + * Desc: Cleans up and removes a whole multi stack. It does not free or + * touch any individual easy handles in any way. We need to define + * in what state those handles will be if this function is called + * in the middle of a transfer. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_cleanup(CURLM *multi_handle); + +/* + * Name: curl_multi_info_read() + * + * Desc: Ask the multi handle if there's any messages/informationals from + * the individual transfers. Messages include informationals such as + * error code from the transfer or just the fact that a transfer is + * completed. More details on these should be written down as well. + * + * Repeated calls to this function will return a new struct each + * time, until a special "end of msgs" struct is returned as a signal + * that there is no more to get at this point. + * + * The data the returned pointer points to will not survive calling + * curl_multi_cleanup(). + * + * The 'CURLMsg' struct is meant to be very simple and only contain + * very basic informations. If more involved information is wanted, + * we will provide the particular "transfer handle" in that struct + * and that should/could/would be used in subsequent + * curl_easy_getinfo() calls (or similar). The point being that we + * must never expose complex structs to applications, as then we'll + * undoubtably get backwards compatibility problems in the future. + * + * Returns: A pointer to a filled-in struct, or NULL if it failed or ran out + * of structs. It also writes the number of messages left in the + * queue (after this read) in the integer the second argument points + * to. + */ +CURL_EXTERN CURLMsg *curl_multi_info_read(CURLM *multi_handle, + int *msgs_in_queue); + +/* + * Name: curl_multi_strerror() + * + * Desc: The curl_multi_strerror function may be used to turn a CURLMcode + * value into the equivalent human readable error string. This is + * useful for printing meaningful error messages. + * + * Returns: A pointer to a zero-terminated error message. + */ +CURL_EXTERN const char *curl_multi_strerror(CURLMcode); + +/* + * Name: curl_multi_socket() and + * curl_multi_socket_all() + * + * Desc: An alternative version of curl_multi_perform() that allows the + * application to pass in one of the file descriptors that have been + * detected to have "action" on them and let libcurl perform. + * See man page for details. + */ +#define CURL_POLL_NONE 0 +#define CURL_POLL_IN 1 +#define CURL_POLL_OUT 2 +#define CURL_POLL_INOUT 3 +#define CURL_POLL_REMOVE 4 + +#define CURL_SOCKET_TIMEOUT CURL_SOCKET_BAD + +#define CURL_CSELECT_IN 0x01 +#define CURL_CSELECT_OUT 0x02 +#define CURL_CSELECT_ERR 0x04 + +typedef int (*curl_socket_callback)(CURL *easy, /* easy handle */ + curl_socket_t s, /* socket */ + int what, /* see above */ + void *userp, /* private callback + pointer */ + void *socketp); /* private socket + pointer */ +/* + * Name: curl_multi_timer_callback + * + * Desc: Called by libcurl whenever the library detects a change in the + * maximum number of milliseconds the app is allowed to wait before + * curl_multi_socket() or curl_multi_perform() must be called + * (to allow libcurl's timed events to take place). + * + * Returns: The callback should return zero. + */ +typedef int (*curl_multi_timer_callback)(CURLM *multi, /* multi handle */ + long timeout_ms, /* see above */ + void *userp); /* private callback + pointer */ + +CURL_EXTERN CURLMcode curl_multi_socket(CURLM *multi_handle, curl_socket_t s, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_action(CURLM *multi_handle, + curl_socket_t s, + int ev_bitmask, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_all(CURLM *multi_handle, + int *running_handles); + +#ifndef CURL_ALLOW_OLD_MULTI_SOCKET +/* This macro below was added in 7.16.3 to push users who recompile to use + the new curl_multi_socket_action() instead of the old curl_multi_socket() +*/ +#define curl_multi_socket(x,y,z) curl_multi_socket_action(x,y,0,z) +#endif + +/* + * Name: curl_multi_timeout() + * + * Desc: Returns the maximum number of milliseconds the app is allowed to + * wait before curl_multi_socket() or curl_multi_perform() must be + * called (to allow libcurl's timed events to take place). + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_timeout(CURLM *multi_handle, + long *milliseconds); + +#undef CINIT /* re-using the same name as in curl.h */ + +#ifdef CURL_ISOCPP +#define CINIT(name,type,num) CURLMOPT_ ## name = CURLOPTTYPE_ ## type + num +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLMOPT_/**/name = type + number +#endif + +typedef enum { + /* This is the socket callback function pointer */ + CINIT(SOCKETFUNCTION, FUNCTIONPOINT, 1), + + /* This is the argument passed to the socket callback */ + CINIT(SOCKETDATA, OBJECTPOINT, 2), + + /* set to 1 to enable pipelining for this multi handle */ + CINIT(PIPELINING, LONG, 3), + + /* This is the timer callback function pointer */ + CINIT(TIMERFUNCTION, FUNCTIONPOINT, 4), + + /* This is the argument passed to the timer callback */ + CINIT(TIMERDATA, OBJECTPOINT, 5), + + /* maximum number of entries in the connection cache */ + CINIT(MAXCONNECTS, LONG, 6), + + /* maximum number of (pipelining) connections to one host */ + CINIT(MAX_HOST_CONNECTIONS, LONG, 7), + + /* maximum number of requests in a pipeline */ + CINIT(MAX_PIPELINE_LENGTH, LONG, 8), + + /* a connection with a content-length longer than this + will not be considered for pipelining */ + CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9), + + /* a connection with a chunk length longer than this + will not be considered for pipelining */ + CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10), + + /* a list of site names(+port) that are blacklisted from + pipelining */ + CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11), + + /* a list of server types that are blacklisted from + pipelining */ + CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12), + + /* maximum number of open connections in total */ + CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13), + + /* This is the server push callback function pointer */ + CINIT(PUSHFUNCTION, FUNCTIONPOINT, 14), + + /* This is the argument passed to the server push callback */ + CINIT(PUSHDATA, OBJECTPOINT, 15), + + CURLMOPT_LASTENTRY /* the last unused */ +} CURLMoption; + + +/* + * Name: curl_multi_setopt() + * + * Desc: Sets options for the multi handle. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_setopt(CURLM *multi_handle, + CURLMoption option, ...); + + +/* + * Name: curl_multi_assign() + * + * Desc: This function sets an association in the multi handle between the + * given socket and a private pointer of the application. This is + * (only) useful for curl_multi_socket uses. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle, + curl_socket_t sockfd, void *sockp); + + +/* + * Name: curl_push_callback + * + * Desc: This callback gets called when a new stream is being pushed by the + * server. It approves or denies the new stream. + * + * Returns: CURL_PUSH_OK or CURL_PUSH_DENY. + */ +#define CURL_PUSH_OK 0 +#define CURL_PUSH_DENY 1 + +struct curl_pushheaders; /* forward declaration only */ + +CURL_EXTERN char *curl_pushheader_bynum(struct curl_pushheaders *h, + size_t num); +CURL_EXTERN char *curl_pushheader_byname(struct curl_pushheaders *h, + const char *name); + +typedef int (*curl_push_callback)(CURL *parent, + CURL *easy, + size_t num_headers, + struct curl_pushheaders *headers, + void *userp); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif diff --git a/deps/libcurl/include/curl/stdcheaders.h b/deps/libcurl/include/curl/stdcheaders.h new file mode 100644 index 0000000000000000000000000000000000000000..ad82ef6335d6167aecf8b9e49d68c462ea028bfd --- /dev/null +++ b/deps/libcurl/include/curl/stdcheaders.h @@ -0,0 +1,33 @@ +#ifndef __STDC_HEADERS_H +#define __STDC_HEADERS_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2010, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include + +size_t fread (void *, size_t, size_t, FILE *); +size_t fwrite (const void *, size_t, size_t, FILE *); + +int strcasecmp(const char *, const char *); +int strncasecmp(const char *, const char *, size_t); + +#endif /* __STDC_HEADERS_H */ diff --git a/deps/libcurl/include/curl/typecheck-gcc.h b/deps/libcurl/include/curl/typecheck-gcc.h new file mode 100644 index 0000000000000000000000000000000000000000..2e24db0ff59c40b137ec89a9d8a3f571ed4791b2 --- /dev/null +++ b/deps/libcurl/include/curl/typecheck-gcc.h @@ -0,0 +1,622 @@ +#ifndef __CURL_TYPECHECK_GCC_H +#define __CURL_TYPECHECK_GCC_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* wraps curl_easy_setopt() with typechecking */ + +/* To add a new kind of warning, add an + * if(_curl_is_sometype_option(_curl_opt)) + * if(!_curl_is_sometype(value)) + * _curl_easy_setopt_err_sometype(); + * block and define _curl_is_sometype_option, _curl_is_sometype and + * _curl_easy_setopt_err_sometype below + * + * NOTE: We use two nested 'if' statements here instead of the && operator, in + * order to work around gcc bug #32061. It affects only gcc 4.3.x/4.4.x + * when compiling with -Wlogical-op. + * + * To add an option that uses the same type as an existing option, you'll just + * need to extend the appropriate _curl_*_option macro + */ +#define curl_easy_setopt(handle, option, value) \ +__extension__ ({ \ + __typeof__ (option) _curl_opt = option; \ + if(__builtin_constant_p(_curl_opt)) { \ + if(_curl_is_long_option(_curl_opt)) \ + if(!_curl_is_long(value)) \ + _curl_easy_setopt_err_long(); \ + if(_curl_is_off_t_option(_curl_opt)) \ + if(!_curl_is_off_t(value)) \ + _curl_easy_setopt_err_curl_off_t(); \ + if(_curl_is_string_option(_curl_opt)) \ + if(!_curl_is_string(value)) \ + _curl_easy_setopt_err_string(); \ + if(_curl_is_write_cb_option(_curl_opt)) \ + if(!_curl_is_write_cb(value)) \ + _curl_easy_setopt_err_write_callback(); \ + if((_curl_opt) == CURLOPT_READFUNCTION) \ + if(!_curl_is_read_cb(value)) \ + _curl_easy_setopt_err_read_cb(); \ + if((_curl_opt) == CURLOPT_IOCTLFUNCTION) \ + if(!_curl_is_ioctl_cb(value)) \ + _curl_easy_setopt_err_ioctl_cb(); \ + if((_curl_opt) == CURLOPT_SOCKOPTFUNCTION) \ + if(!_curl_is_sockopt_cb(value)) \ + _curl_easy_setopt_err_sockopt_cb(); \ + if((_curl_opt) == CURLOPT_OPENSOCKETFUNCTION) \ + if(!_curl_is_opensocket_cb(value)) \ + _curl_easy_setopt_err_opensocket_cb(); \ + if((_curl_opt) == CURLOPT_PROGRESSFUNCTION) \ + if(!_curl_is_progress_cb(value)) \ + _curl_easy_setopt_err_progress_cb(); \ + if((_curl_opt) == CURLOPT_DEBUGFUNCTION) \ + if(!_curl_is_debug_cb(value)) \ + _curl_easy_setopt_err_debug_cb(); \ + if((_curl_opt) == CURLOPT_SSL_CTX_FUNCTION) \ + if(!_curl_is_ssl_ctx_cb(value)) \ + _curl_easy_setopt_err_ssl_ctx_cb(); \ + if(_curl_is_conv_cb_option(_curl_opt)) \ + if(!_curl_is_conv_cb(value)) \ + _curl_easy_setopt_err_conv_cb(); \ + if((_curl_opt) == CURLOPT_SEEKFUNCTION) \ + if(!_curl_is_seek_cb(value)) \ + _curl_easy_setopt_err_seek_cb(); \ + if(_curl_is_cb_data_option(_curl_opt)) \ + if(!_curl_is_cb_data(value)) \ + _curl_easy_setopt_err_cb_data(); \ + if((_curl_opt) == CURLOPT_ERRORBUFFER) \ + if(!_curl_is_error_buffer(value)) \ + _curl_easy_setopt_err_error_buffer(); \ + if((_curl_opt) == CURLOPT_STDERR) \ + if(!_curl_is_FILE(value)) \ + _curl_easy_setopt_err_FILE(); \ + if(_curl_is_postfields_option(_curl_opt)) \ + if(!_curl_is_postfields(value)) \ + _curl_easy_setopt_err_postfields(); \ + if((_curl_opt) == CURLOPT_HTTPPOST) \ + if(!_curl_is_arr((value), struct curl_httppost)) \ + _curl_easy_setopt_err_curl_httpost(); \ + if(_curl_is_slist_option(_curl_opt)) \ + if(!_curl_is_arr((value), struct curl_slist)) \ + _curl_easy_setopt_err_curl_slist(); \ + if((_curl_opt) == CURLOPT_SHARE) \ + if(!_curl_is_ptr((value), CURLSH)) \ + _curl_easy_setopt_err_CURLSH(); \ + } \ + curl_easy_setopt(handle, _curl_opt, value); \ +}) + +/* wraps curl_easy_getinfo() with typechecking */ +/* FIXME: don't allow const pointers */ +#define curl_easy_getinfo(handle, info, arg) \ +__extension__ ({ \ + __typeof__ (info) _curl_info = info; \ + if(__builtin_constant_p(_curl_info)) { \ + if(_curl_is_string_info(_curl_info)) \ + if(!_curl_is_arr((arg), char *)) \ + _curl_easy_getinfo_err_string(); \ + if(_curl_is_long_info(_curl_info)) \ + if(!_curl_is_arr((arg), long)) \ + _curl_easy_getinfo_err_long(); \ + if(_curl_is_double_info(_curl_info)) \ + if(!_curl_is_arr((arg), double)) \ + _curl_easy_getinfo_err_double(); \ + if(_curl_is_slist_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_slist *)) \ + _curl_easy_getinfo_err_curl_slist(); \ + } \ + curl_easy_getinfo(handle, _curl_info, arg); \ +}) + +/* TODO: typechecking for curl_share_setopt() and curl_multi_setopt(), + * for now just make sure that the functions are called with three + * arguments + */ +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) + + +/* the actual warnings, triggered by calling the _curl_easy_setopt_err* + * functions */ + +/* To define a new warning, use _CURL_WARNING(identifier, "message") */ +#define _CURL_WARNING(id, message) \ + static void __attribute__((__warning__(message))) \ + __attribute__((__unused__)) __attribute__((__noinline__)) \ + id(void) { __asm__(""); } + +_CURL_WARNING(_curl_easy_setopt_err_long, + "curl_easy_setopt expects a long argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_off_t, + "curl_easy_setopt expects a curl_off_t argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_string, + "curl_easy_setopt expects a " + "string (char* or char[]) argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_write_callback, + "curl_easy_setopt expects a curl_write_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_read_cb, + "curl_easy_setopt expects a curl_read_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ioctl_cb, + "curl_easy_setopt expects a curl_ioctl_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_sockopt_cb, + "curl_easy_setopt expects a curl_sockopt_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_opensocket_cb, + "curl_easy_setopt expects a " + "curl_opensocket_callback argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_progress_cb, + "curl_easy_setopt expects a curl_progress_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_debug_cb, + "curl_easy_setopt expects a curl_debug_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ssl_ctx_cb, + "curl_easy_setopt expects a curl_ssl_ctx_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_conv_cb, + "curl_easy_setopt expects a curl_conv_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_seek_cb, + "curl_easy_setopt expects a curl_seek_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_cb_data, + "curl_easy_setopt expects a " + "private data pointer as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_error_buffer, + "curl_easy_setopt expects a " + "char buffer of CURL_ERROR_SIZE as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_FILE, + "curl_easy_setopt expects a FILE* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_postfields, + "curl_easy_setopt expects a void* or char* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_httpost, + "curl_easy_setopt expects a struct curl_httppost* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_slist, + "curl_easy_setopt expects a struct curl_slist* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_CURLSH, + "curl_easy_setopt expects a CURLSH* argument for this option") + +_CURL_WARNING(_curl_easy_getinfo_err_string, + "curl_easy_getinfo expects a pointer to char * for this info") +_CURL_WARNING(_curl_easy_getinfo_err_long, + "curl_easy_getinfo expects a pointer to long for this info") +_CURL_WARNING(_curl_easy_getinfo_err_double, + "curl_easy_getinfo expects a pointer to double for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_slist, + "curl_easy_getinfo expects a pointer to struct curl_slist * for this info") + +/* groups of curl_easy_setops options that take the same type of argument */ + +/* To add a new option to one of the groups, just add + * (option) == CURLOPT_SOMETHING + * to the or-expression. If the option takes a long or curl_off_t, you don't + * have to do anything + */ + +/* evaluates to true if option takes a long argument */ +#define _curl_is_long_option(option) \ + (0 < (option) && (option) < CURLOPTTYPE_OBJECTPOINT) + +#define _curl_is_off_t_option(option) \ + ((option) > CURLOPTTYPE_OFF_T) + +/* evaluates to true if option takes a char* argument */ +#define _curl_is_string_option(option) \ + ((option) == CURLOPT_ACCEPT_ENCODING || \ + (option) == CURLOPT_CAINFO || \ + (option) == CURLOPT_CAPATH || \ + (option) == CURLOPT_COOKIE || \ + (option) == CURLOPT_COOKIEFILE || \ + (option) == CURLOPT_COOKIEJAR || \ + (option) == CURLOPT_COOKIELIST || \ + (option) == CURLOPT_CRLFILE || \ + (option) == CURLOPT_CUSTOMREQUEST || \ + (option) == CURLOPT_DEFAULT_PROTOCOL || \ + (option) == CURLOPT_DNS_INTERFACE || \ + (option) == CURLOPT_DNS_LOCAL_IP4 || \ + (option) == CURLOPT_DNS_LOCAL_IP6 || \ + (option) == CURLOPT_DNS_SERVERS || \ + (option) == CURLOPT_EGDSOCKET || \ + (option) == CURLOPT_FTPPORT || \ + (option) == CURLOPT_FTP_ACCOUNT || \ + (option) == CURLOPT_FTP_ALTERNATIVE_TO_USER || \ + (option) == CURLOPT_INTERFACE || \ + (option) == CURLOPT_ISSUERCERT || \ + (option) == CURLOPT_KEYPASSWD || \ + (option) == CURLOPT_KRBLEVEL || \ + (option) == CURLOPT_LOGIN_OPTIONS || \ + (option) == CURLOPT_MAIL_AUTH || \ + (option) == CURLOPT_MAIL_FROM || \ + (option) == CURLOPT_NETRC_FILE || \ + (option) == CURLOPT_NOPROXY || \ + (option) == CURLOPT_PASSWORD || \ + (option) == CURLOPT_PINNEDPUBLICKEY || \ + (option) == CURLOPT_PROXY || \ + (option) == CURLOPT_PROXYPASSWORD || \ + (option) == CURLOPT_PROXYUSERNAME || \ + (option) == CURLOPT_PROXYUSERPWD || \ + (option) == CURLOPT_PROXY_SERVICE_NAME || \ + (option) == CURLOPT_RANDOM_FILE || \ + (option) == CURLOPT_RANGE || \ + (option) == CURLOPT_REFERER || \ + (option) == CURLOPT_RTSP_SESSION_ID || \ + (option) == CURLOPT_RTSP_STREAM_URI || \ + (option) == CURLOPT_RTSP_TRANSPORT || \ + (option) == CURLOPT_SERVICE_NAME || \ + (option) == CURLOPT_SOCKS5_GSSAPI_SERVICE || \ + (option) == CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 || \ + (option) == CURLOPT_SSH_KNOWNHOSTS || \ + (option) == CURLOPT_SSH_PRIVATE_KEYFILE || \ + (option) == CURLOPT_SSH_PUBLIC_KEYFILE || \ + (option) == CURLOPT_SSLCERT || \ + (option) == CURLOPT_SSLCERTTYPE || \ + (option) == CURLOPT_SSLENGINE || \ + (option) == CURLOPT_SSLKEY || \ + (option) == CURLOPT_SSLKEYTYPE || \ + (option) == CURLOPT_SSL_CIPHER_LIST || \ + (option) == CURLOPT_TLSAUTH_PASSWORD || \ + (option) == CURLOPT_TLSAUTH_TYPE || \ + (option) == CURLOPT_TLSAUTH_USERNAME || \ + (option) == CURLOPT_UNIX_SOCKET_PATH || \ + (option) == CURLOPT_URL || \ + (option) == CURLOPT_USERAGENT || \ + (option) == CURLOPT_USERNAME || \ + (option) == CURLOPT_USERPWD || \ + (option) == CURLOPT_XOAUTH2_BEARER || \ + 0) + +/* evaluates to true if option takes a curl_write_callback argument */ +#define _curl_is_write_cb_option(option) \ + ((option) == CURLOPT_HEADERFUNCTION || \ + (option) == CURLOPT_WRITEFUNCTION) + +/* evaluates to true if option takes a curl_conv_callback argument */ +#define _curl_is_conv_cb_option(option) \ + ((option) == CURLOPT_CONV_TO_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_UTF8_FUNCTION) + +/* evaluates to true if option takes a data argument to pass to a callback */ +#define _curl_is_cb_data_option(option) \ + ((option) == CURLOPT_CHUNK_DATA || \ + (option) == CURLOPT_CLOSESOCKETDATA || \ + (option) == CURLOPT_DEBUGDATA || \ + (option) == CURLOPT_FNMATCH_DATA || \ + (option) == CURLOPT_HEADERDATA || \ + (option) == CURLOPT_INTERLEAVEDATA || \ + (option) == CURLOPT_IOCTLDATA || \ + (option) == CURLOPT_OPENSOCKETDATA || \ + (option) == CURLOPT_PRIVATE || \ + (option) == CURLOPT_PROGRESSDATA || \ + (option) == CURLOPT_READDATA || \ + (option) == CURLOPT_SEEKDATA || \ + (option) == CURLOPT_SOCKOPTDATA || \ + (option) == CURLOPT_SSH_KEYDATA || \ + (option) == CURLOPT_SSL_CTX_DATA || \ + (option) == CURLOPT_WRITEDATA || \ + 0) + +/* evaluates to true if option takes a POST data argument (void* or char*) */ +#define _curl_is_postfields_option(option) \ + ((option) == CURLOPT_POSTFIELDS || \ + (option) == CURLOPT_COPYPOSTFIELDS || \ + 0) + +/* evaluates to true if option takes a struct curl_slist * argument */ +#define _curl_is_slist_option(option) \ + ((option) == CURLOPT_HTTP200ALIASES || \ + (option) == CURLOPT_HTTPHEADER || \ + (option) == CURLOPT_MAIL_RCPT || \ + (option) == CURLOPT_POSTQUOTE || \ + (option) == CURLOPT_PREQUOTE || \ + (option) == CURLOPT_PROXYHEADER || \ + (option) == CURLOPT_QUOTE || \ + (option) == CURLOPT_RESOLVE || \ + (option) == CURLOPT_TELNETOPTIONS || \ + 0) + +/* groups of curl_easy_getinfo infos that take the same type of argument */ + +/* evaluates to true if info expects a pointer to char * argument */ +#define _curl_is_string_info(info) \ + (CURLINFO_STRING < (info) && (info) < CURLINFO_LONG) + +/* evaluates to true if info expects a pointer to long argument */ +#define _curl_is_long_info(info) \ + (CURLINFO_LONG < (info) && (info) < CURLINFO_DOUBLE) + +/* evaluates to true if info expects a pointer to double argument */ +#define _curl_is_double_info(info) \ + (CURLINFO_DOUBLE < (info) && (info) < CURLINFO_SLIST) + +/* true if info expects a pointer to struct curl_slist * argument */ +#define _curl_is_slist_info(info) \ + (CURLINFO_SLIST < (info)) + + +/* typecheck helpers -- check whether given expression has requested type*/ + +/* For pointers, you can use the _curl_is_ptr/_curl_is_arr macros, + * otherwise define a new macro. Search for __builtin_types_compatible_p + * in the GCC manual. + * NOTE: these macros MUST NOT EVALUATE their arguments! The argument is + * the actual expression passed to the curl_easy_setopt macro. This + * means that you can only apply the sizeof and __typeof__ operators, no + * == or whatsoever. + */ + +/* XXX: should evaluate to true iff expr is a pointer */ +#define _curl_is_any_ptr(expr) \ + (sizeof(expr) == sizeof(void*)) + +/* evaluates to true if expr is NULL */ +/* XXX: must not evaluate expr, so this check is not accurate */ +#define _curl_is_NULL(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), __typeof__(NULL))) + +/* evaluates to true if expr is type*, const type* or NULL */ +#define _curl_is_ptr(expr, type) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), type *) || \ + __builtin_types_compatible_p(__typeof__(expr), const type *)) + +/* evaluates to true if expr is one of type[], type*, NULL or const type* */ +#define _curl_is_arr(expr, type) \ + (_curl_is_ptr((expr), type) || \ + __builtin_types_compatible_p(__typeof__(expr), type [])) + +/* evaluates to true if expr is a string */ +#define _curl_is_string(expr) \ + (_curl_is_arr((expr), char) || \ + _curl_is_arr((expr), signed char) || \ + _curl_is_arr((expr), unsigned char)) + +/* evaluates to true if expr is a long (no matter the signedness) + * XXX: for now, int is also accepted (and therefore short and char, which + * are promoted to int when passed to a variadic function) */ +#define _curl_is_long(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), long) || \ + __builtin_types_compatible_p(__typeof__(expr), signed long) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned long) || \ + __builtin_types_compatible_p(__typeof__(expr), int) || \ + __builtin_types_compatible_p(__typeof__(expr), signed int) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned int) || \ + __builtin_types_compatible_p(__typeof__(expr), short) || \ + __builtin_types_compatible_p(__typeof__(expr), signed short) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned short) || \ + __builtin_types_compatible_p(__typeof__(expr), char) || \ + __builtin_types_compatible_p(__typeof__(expr), signed char) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned char)) + +/* evaluates to true if expr is of type curl_off_t */ +#define _curl_is_off_t(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), curl_off_t)) + +/* evaluates to true if expr is abuffer suitable for CURLOPT_ERRORBUFFER */ +/* XXX: also check size of an char[] array? */ +#define _curl_is_error_buffer(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), char *) || \ + __builtin_types_compatible_p(__typeof__(expr), char[])) + +/* evaluates to true if expr is of type (const) void* or (const) FILE* */ +#if 0 +#define _curl_is_cb_data(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_ptr((expr), FILE)) +#else /* be less strict */ +#define _curl_is_cb_data(expr) \ + _curl_is_any_ptr(expr) +#endif + +/* evaluates to true if expr is of type FILE* */ +#define _curl_is_FILE(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), FILE *)) + +/* evaluates to true if expr can be passed as POST data (void* or char*) */ +#define _curl_is_postfields(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_arr((expr), char)) + +/* FIXME: the whole callback checking is messy... + * The idea is to tolerate char vs. void and const vs. not const + * pointers in arguments at least + */ +/* helper: __builtin_types_compatible_p distinguishes between functions and + * function pointers, hide it */ +#define _curl_callback_compatible(func, type) \ + (__builtin_types_compatible_p(__typeof__(func), type) || \ + __builtin_types_compatible_p(__typeof__(func), type*)) + +/* evaluates to true if expr is of type curl_read_callback or "similar" */ +#define _curl_is_read_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), __typeof__(fread)) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_read_callback) || \ + _curl_callback_compatible((expr), _curl_read_callback1) || \ + _curl_callback_compatible((expr), _curl_read_callback2) || \ + _curl_callback_compatible((expr), _curl_read_callback3) || \ + _curl_callback_compatible((expr), _curl_read_callback4) || \ + _curl_callback_compatible((expr), _curl_read_callback5) || \ + _curl_callback_compatible((expr), _curl_read_callback6)) +typedef size_t (_curl_read_callback1)(char *, size_t, size_t, void*); +typedef size_t (_curl_read_callback2)(char *, size_t, size_t, const void*); +typedef size_t (_curl_read_callback3)(char *, size_t, size_t, FILE*); +typedef size_t (_curl_read_callback4)(void *, size_t, size_t, void*); +typedef size_t (_curl_read_callback5)(void *, size_t, size_t, const void*); +typedef size_t (_curl_read_callback6)(void *, size_t, size_t, FILE*); + +/* evaluates to true if expr is of type curl_write_callback or "similar" */ +#define _curl_is_write_cb(expr) \ + (_curl_is_read_cb(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), __typeof__(fwrite)) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_write_callback) || \ + _curl_callback_compatible((expr), _curl_write_callback1) || \ + _curl_callback_compatible((expr), _curl_write_callback2) || \ + _curl_callback_compatible((expr), _curl_write_callback3) || \ + _curl_callback_compatible((expr), _curl_write_callback4) || \ + _curl_callback_compatible((expr), _curl_write_callback5) || \ + _curl_callback_compatible((expr), _curl_write_callback6)) +typedef size_t (_curl_write_callback1)(const char *, size_t, size_t, void*); +typedef size_t (_curl_write_callback2)(const char *, size_t, size_t, + const void*); +typedef size_t (_curl_write_callback3)(const char *, size_t, size_t, FILE*); +typedef size_t (_curl_write_callback4)(const void *, size_t, size_t, void*); +typedef size_t (_curl_write_callback5)(const void *, size_t, size_t, + const void*); +typedef size_t (_curl_write_callback6)(const void *, size_t, size_t, FILE*); + +/* evaluates to true if expr is of type curl_ioctl_callback or "similar" */ +#define _curl_is_ioctl_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_ioctl_callback) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback1) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback2) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback3) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback4)) +typedef curlioerr (_curl_ioctl_callback1)(CURL *, int, void*); +typedef curlioerr (_curl_ioctl_callback2)(CURL *, int, const void*); +typedef curlioerr (_curl_ioctl_callback3)(CURL *, curliocmd, void*); +typedef curlioerr (_curl_ioctl_callback4)(CURL *, curliocmd, const void*); + +/* evaluates to true if expr is of type curl_sockopt_callback or "similar" */ +#define _curl_is_sockopt_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_sockopt_callback) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback1) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback2)) +typedef int (_curl_sockopt_callback1)(void *, curl_socket_t, curlsocktype); +typedef int (_curl_sockopt_callback2)(const void *, curl_socket_t, + curlsocktype); + +/* evaluates to true if expr is of type curl_opensocket_callback or + "similar" */ +#define _curl_is_opensocket_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_opensocket_callback) ||\ + _curl_callback_compatible((expr), _curl_opensocket_callback1) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback2) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback3) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback4)) +typedef curl_socket_t (_curl_opensocket_callback1) + (void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (_curl_opensocket_callback2) + (void *, curlsocktype, const struct curl_sockaddr *); +typedef curl_socket_t (_curl_opensocket_callback3) + (const void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (_curl_opensocket_callback4) + (const void *, curlsocktype, const struct curl_sockaddr *); + +/* evaluates to true if expr is of type curl_progress_callback or "similar" */ +#define _curl_is_progress_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_progress_callback) || \ + _curl_callback_compatible((expr), _curl_progress_callback1) || \ + _curl_callback_compatible((expr), _curl_progress_callback2)) +typedef int (_curl_progress_callback1)(void *, + double, double, double, double); +typedef int (_curl_progress_callback2)(const void *, + double, double, double, double); + +/* evaluates to true if expr is of type curl_debug_callback or "similar" */ +#define _curl_is_debug_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_debug_callback) || \ + _curl_callback_compatible((expr), _curl_debug_callback1) || \ + _curl_callback_compatible((expr), _curl_debug_callback2) || \ + _curl_callback_compatible((expr), _curl_debug_callback3) || \ + _curl_callback_compatible((expr), _curl_debug_callback4) || \ + _curl_callback_compatible((expr), _curl_debug_callback5) || \ + _curl_callback_compatible((expr), _curl_debug_callback6) || \ + _curl_callback_compatible((expr), _curl_debug_callback7) || \ + _curl_callback_compatible((expr), _curl_debug_callback8)) +typedef int (_curl_debug_callback1) (CURL *, + curl_infotype, char *, size_t, void *); +typedef int (_curl_debug_callback2) (CURL *, + curl_infotype, char *, size_t, const void *); +typedef int (_curl_debug_callback3) (CURL *, + curl_infotype, const char *, size_t, void *); +typedef int (_curl_debug_callback4) (CURL *, + curl_infotype, const char *, size_t, const void *); +typedef int (_curl_debug_callback5) (CURL *, + curl_infotype, unsigned char *, size_t, void *); +typedef int (_curl_debug_callback6) (CURL *, + curl_infotype, unsigned char *, size_t, const void *); +typedef int (_curl_debug_callback7) (CURL *, + curl_infotype, const unsigned char *, size_t, void *); +typedef int (_curl_debug_callback8) (CURL *, + curl_infotype, const unsigned char *, size_t, const void *); + +/* evaluates to true if expr is of type curl_ssl_ctx_callback or "similar" */ +/* this is getting even messier... */ +#define _curl_is_ssl_ctx_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_ssl_ctx_callback) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback1) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback2) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback3) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback4) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback5) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback6) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback7) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback8)) +typedef CURLcode (_curl_ssl_ctx_callback1)(CURL *, void *, void *); +typedef CURLcode (_curl_ssl_ctx_callback2)(CURL *, void *, const void *); +typedef CURLcode (_curl_ssl_ctx_callback3)(CURL *, const void *, void *); +typedef CURLcode (_curl_ssl_ctx_callback4)(CURL *, const void *, const void *); +#ifdef HEADER_SSL_H +/* hack: if we included OpenSSL's ssl.h, we know about SSL_CTX + * this will of course break if we're included before OpenSSL headers... + */ +typedef CURLcode (_curl_ssl_ctx_callback5)(CURL *, SSL_CTX, void *); +typedef CURLcode (_curl_ssl_ctx_callback6)(CURL *, SSL_CTX, const void *); +typedef CURLcode (_curl_ssl_ctx_callback7)(CURL *, const SSL_CTX, void *); +typedef CURLcode (_curl_ssl_ctx_callback8)(CURL *, const SSL_CTX, + const void *); +#else +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback5; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback6; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback7; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback8; +#endif + +/* evaluates to true if expr is of type curl_conv_callback or "similar" */ +#define _curl_is_conv_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_conv_callback) || \ + _curl_callback_compatible((expr), _curl_conv_callback1) || \ + _curl_callback_compatible((expr), _curl_conv_callback2) || \ + _curl_callback_compatible((expr), _curl_conv_callback3) || \ + _curl_callback_compatible((expr), _curl_conv_callback4)) +typedef CURLcode (*_curl_conv_callback1)(char *, size_t length); +typedef CURLcode (*_curl_conv_callback2)(const char *, size_t length); +typedef CURLcode (*_curl_conv_callback3)(void *, size_t length); +typedef CURLcode (*_curl_conv_callback4)(const void *, size_t length); + +/* evaluates to true if expr is of type curl_seek_callback or "similar" */ +#define _curl_is_seek_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_seek_callback) || \ + _curl_callback_compatible((expr), _curl_seek_callback1) || \ + _curl_callback_compatible((expr), _curl_seek_callback2)) +typedef CURLcode (*_curl_seek_callback1)(void *, curl_off_t, int); +typedef CURLcode (*_curl_seek_callback2)(const void *, curl_off_t, int); + + +#endif /* __CURL_TYPECHECK_GCC_H */ diff --git a/deps/libcurl/lib/libcurl.a b/deps/libcurl/lib/libcurl.a new file mode 100644 index 0000000000000000000000000000000000000000..c63ac70946dddefb5b3b5c55cf82d693b731cad6 Binary files /dev/null and b/deps/libcurl/lib/libcurl.a differ diff --git a/documentation20/webdocs/assets/connector.png b/documentation20/webdocs/assets/connector.png index c30a50a8302cc5cad15471e16d064a36c1fa37ea..6030bd73f51123615eabacfa5d734918559ce3d9 100644 Binary files a/documentation20/webdocs/assets/connector.png and b/documentation20/webdocs/assets/connector.png differ diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md index 766a428f1bfa8559dfb86701195868a8b6821ebc..20f9bcb7307c68f1330519d8f6ed249419bf46db 100644 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md @@ -128,5 +128,18 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 ## [培训和FAQ](https://www.taosdata.com/cn/faq) -- [FAQ](https://www.taosdata.com/cn/documentation20/faq)：常见问题与答案 -- [应用案列](https://www.taosdata.com/cn/blog/?categories=4)：一些使用实例来解释如何使用TDengine + + diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/webdocs/markdowndocs/Evaluation-ch.md index a92f97a8d9dfc9a47d5554daa076f4d2f0774c92..fa6cec6e488d144a7009dc52772987380d5065da 100644 --- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md +++ b/documentation20/webdocs/markdowndocs/Evaluation-ch.md @@ -59,5 +59,3 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 |要求运维学习成本可控| | | √ |同上。| |要求市场有大量人才储备| √ | | |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务。| -## TDengine 性能指标介绍和验证方法 - diff --git a/documentation20/webdocs/markdowndocs/Getting Started-ch.md b/documentation20/webdocs/markdowndocs/Getting Started-ch.md index b63bfff6c999340882c665fe0f5516a72dd62a72..b53c014ba64fb203cb8e6943297fe6117034385c 100644 --- a/documentation20/webdocs/markdowndocs/Getting Started-ch.md +++ b/documentation20/webdocs/markdowndocs/Getting Started-ch.md @@ -2,35 +2,7 @@ ## 快捷安装 -TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。 - -**应用驱动** - -如果应用在Windows和Linux上运行，可使用C/C++/C#/JAVA/Python/Go/Node.js接口连接服务器。如果应用在Mac上运行，目前可以使用RESTful接口连接服务器。 - -**CPU** - -CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 - -**服务器** - -目前TDengine服务器可以运行在以下平台上： - -| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **Win64/32** | **macOS** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** | -| -------------- | --------------------- | ------------------------ | --------------- | ------------ | --------- | --------------- | ------------------------- | --------------------- | -| X64 | ● | ● | | ○/○ | ○ | ○ | ● | ● | -| 树莓派ARM32 | | ● | ● | | | | | | -| 龙芯MIPS64 | | | ● | | | | | | -| 鲲鹏 ARM64 | | ○ | ○ | | | | ● | | -| 申威 Alpha64 | | | ○ | | | ● | | | -| 飞腾ARM64 | | | ○优麒麟 | | | | | | -| 海光X64 | ● | ● | ● | | | ○ | ● | ● | -| 瑞芯微ARM64/32 | | | ○ | | | | | | -| 全志ARM64/32 | | | ○ | | | | | | -| 炬力ARM64/32 | | | ○ | | | | | | -| TI ARM32 | | | ○ | | | | | | - - +TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 ### 通过源码安装 @@ -42,42 +14,25 @@ CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。 ### 通过安装包安装 -服务器部分，我们提供三种安装包，您可以根据需要选择。TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。 - -- TDengine-server-2.0.9.0-Linux-x64.rpm (4.2M) -- TDengine-server-2.0.9.0-Linux-x64.deb (2.7M) -- TDengine-server-2.0.9.0-Linux-x64.tar.gz (4.5M) - - -客户端部分，Linux安装包如下： - -- TDengine-client-2.0.9.0-Linux-x64.tar.gz(3.0M) -- TDengine-client-2.0.9.0-Windows-x64.exe(2.8M) -- TDengine-client-2.0.9.0-Windows-x86.exe(2.8M) - -报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）： - -- TDengine-alert-2.0.9.0-Linux-x64.tar.gz (8.1M) +TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择： -目前，TDengine 支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装，用`which systemctl`命令来检测系统中是否存在`systemd`包: +- TDengine-server-2.0.10.0-Linux-x64.rpm (4.2M) +- TDengine-server-2.0.10.0-Linux-x64.deb (2.7M) +- TDengine-server-2.0.10.0-Linux-x64.tar.gz (4.5M) -```cmd -which systemctl -``` - -具体的安装过程，请参见TDengine多种安装包的安装和卸载。 +具体的安装过程，请参见TDengine多种安装包的安装和卸载以及视频教程。 ## 轻松启动 安装成功后，用户可使用`systemctl`命令来启动TDengine的服务进程。 -```cmd -systemctl start taosd +```bash +$ systemctl start taosd ``` 检查服务是否正常工作。 -```cmd -systemctl status taosd +```bash +$ systemctl status taosd ``` 如果TDengine服务正常工作，那么您可以通过TDengine的命令行程序`taos`来访问并体验TDengine。 @@ -86,15 +41,24 @@ systemctl status taosd - systemctl命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo - 为更好的获得产品反馈，改善产品，TDengine会采集基本的使用信息，但您可以修改系统配置文件taos.cfg里的配置参数telemetryReporting, 将其设为0，就可将其关闭。 +- TDengine采用FQDN(一般就是hostname)作为节点的ID，为保证正常运行，需要给运行taosd的服务器配置好hostname，在客户端应用运行的机器配置好DNS服务或hosts文件，保证FQDN能够解析。 + +* TDengine 支持在使用[`systemd`](https://en.wikipedia.org/wiki/Systemd)做进程服务管理的linux系统上安装，用`which systemctl`命令来检测系统中是否存在`systemd`包: + + ```bash + $ which systemctl + ``` + + 如果系统中不支持systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 -如果系统中不支持`systemd`，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 + ## TDengine命令行程序 执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。 -```cmd -taos +```bash +$ taos ``` 如果TDengine终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/faq/)来解决终端连接服务端失败的问题）。TDengine终端的提示符号如下： @@ -134,15 +98,15 @@ Query OK, 2 row(s) in set (0.001700s) 示例： -```cmd -taos -h 192.168.0.1 -s "use db; show tables;" +```bash +$ taos -h 192.168.0.1 -s "use db; show tables;" ``` ### 运行SQL命令脚本 TDengine终端可以通过`source`命令来运行SQL命令脚本. -``` +```mysql taos> source ; ``` @@ -157,8 +121,8 @@ taos> source ; 启动TDengine的服务，在Linux终端执行taosdemo -``` -> taosdemo +```bash +$ taosdemo ``` 该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。 @@ -169,33 +133,92 @@ taos> source ; - 查询超级表下记录总条数： -``` -taos>select count(*) from test.meters; +```mysql +taos> select count(*) from test.meters; ``` - 查询10亿条记录的平均值、最大值、最小值等： -``` -taos>select avg(f1), max(f2), min(f3) from test.meters; +```mysql +taos> select avg(f1), max(f2), min(f3) from test.meters; ``` - 查询loc="beijing"的记录总条数： -``` -taos>select count(*) from test.meters where loc="beijing"; +```mysql +taos> select count(*) from test.meters where loc="beijing"; ``` - 查询areaid=10的所有记录的平均值、最大值、最小值等： -``` -taos>select avg(f1), max(f2), min(f3) from test.meters where areaid=10; +```mysql +taos> select avg(f1), max(f2), min(f3) from test.meters where areaid=10; ``` - 对表t10按10s进行平均值、最大值和最小值聚合统计： -``` -taos>select avg(f1), max(f2), min(f3) from test.t10 interval(10s); +```mysql +taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); ``` **Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。 + + +## 客户端和报警模块 + +如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux和Windows安装包如下： + +- TDengine-client-2.0.10.0-Linux-x64.tar.gz(3.0M) +- TDengine-client-2.0.10.0-Windows-x64.exe(2.8M) +- TDengine-client-2.0.10.0-Windows-x86.exe(2.8M) + +报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）： + +- TDengine-alert-2.0.10.0-Linux-x64.tar.gz (8.1M) + + + +## **支持平台列表** + +### TDengine服务器支持的平台列表 + +| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** | +| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | +| X64 | ● | ● | | ○ | ● | ● | +| 树莓派ARM32 | | ● | ● | | | | +| 龙芯MIPS64 | | | ● | | | | +| 鲲鹏 ARM64 | | ○ | ○ | | ● | | +| 申威 Alpha64 | | | ○ | ● | | | +| 飞腾ARM64 | | ○优麒麟 | | | | | +| 海光X64 | ● | ● | ● | ○ | ● | ● | +| 瑞芯微ARM64/32 | | | ○ | | | | +| 全志ARM64/32 | | | ○ | | | | +| 炬力ARM64/32 | | | ○ | | | | +| TI ARM32 | | | ○ | | | | + +注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 + + + +### TDengine客户端和连接器支持的平台列表 + +目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。 + +对照矩阵如下： + +| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** | +| ----------- | --------------- | --------- | --------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ | +| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | +| **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | +| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | +| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● | +| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | +| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | +| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- | +| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● | + +注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 + +请跳转到 [连接器 ](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。 + diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index 5bd29fdb5d30cd365cef79ffc8bc34a936395cc0..dce7819423661a3748c4c5cd4402777e21b16a89 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -8,14 +8,14 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个 不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： -```cmd +```mysql CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4 UPDATE 1; ``` 上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4，允许更新数据。详细的语法及参数请见TAOS SQL 创建库之后，需要使用SQL命令USE将当前库切换过来，例如： -```cmd +```mysql USE power; ``` @@ -28,7 +28,7 @@ USE power; ## 创建超级表 一个物联网系统，往往存在多种类型的设备，比如对于电网，存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合，使用TDengine, 需要对每个类型的数据采集点创建一超级表。以表一中的智能电表为例，可以使用如下的SQL命令创建超级表： -```cmd +```mysql CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); ``` 与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义。第一列必须为时间戳（示例中为ts)，其他列为采集的物理量（示例中为current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 @@ -59,3 +59,5 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。 TDengine建议尽可能采用多列模型，因为插入效率以及存储效率更高。但对于有些场景，一个采集点的采集量的种类经常变化，这个时候，如果采用多列模型，就需要频繁修改超级表的结构定义，让应用变的复杂，这个时候，采用单列模型会显得简单。 + +关于数据建模请参考视频教程。 diff --git a/documentation20/webdocs/markdowndocs/Super Table-ch.md b/documentation20/webdocs/markdowndocs/Super Table-ch.md index 96e7104ab7fe78a0015b596dd31bec6a849f2f6a..e5c77471570a76e608d59a0dca10462315460337 100644 --- a/documentation20/webdocs/markdowndocs/Super Table-ch.md +++ b/documentation20/webdocs/markdowndocs/Super Table-ch.md @@ -161,7 +161,7 @@ SELECT function,… 超级表聚合查询，TDengine目前支持以下聚合\选择函数：sum、count、avg、first、last、min、max、top、bottom，以及针对全部或部分列的投影操作，使用方式与单表查询的计算过程相同。暂不支持其他类型的聚合计算和四则运算。当前所有的函数及计算过程均不支持嵌套的方式进行执行。 - 不使用GROUP BY的查询将会对超级表下所有满足筛选条件的表按时间进行聚合，结果输出默认是按照时间戳单调递增输出，用户可以使用ORDER BY _c0 ASC|DESC选择查询结果时间戳的升降排序；使用GROUP BY 的聚合查询会按照tags进行分组，并对每个组内的数据分别进行聚合，输出结果为各个组的聚合结果，组间的排序可以由ORDER BY 语句指定，每个分组内部，时间序列是单调递增的。 + 不使用GROUP BY的查询将会对超级表下所有满足筛选条件的表按时间进行聚合，结果输出默认是按照时间戳单调递增输出，用户可以使用ORDER BY _c0 ASC|DESC选择查询结果时间戳的升降排序；使用GROUP BY 的聚合查询会按照tags进行分组，并对每个组内的数据分别进行聚合，输出结果为各个组的聚合结果，组间的排序可以由ORDER BY 语句指定，每个分组内部，时间序列是单调递增的。 使用SLIMIT/SOFFSET语句指定组间分页，即指定结果集中输出的最大组数以及对组起始的位置。使用LIMIT/OFFSET语句指定组内分页，即指定结果集中每个组内最多输出多少条记录以及记录起始的位置。 @@ -178,7 +178,7 @@ CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int) ``` -假设有北京，天津和上海三个地区的采集器共4个，温度采集器有3种类型，我们就可以对每个采集器建表如下： +假设有北京，天津和上海三个地区的采集器共4个，温度采集器有3种类型，我们就可以对每个采集器建表如下： ```mysql CREATE TABLE therm1 USING thermometer TAGS (’beijing’, 1); diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index adba39ec1f5275a73b8b473d1ace52748b3e4f4b..b68f60f529d2978366f2ae7d39ff491f60b86f36 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -77,15 +77,13 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic SHOW VARIABLES; ``` - - **使用数据库** - + ```mysql USE db_name; ``` 使用/切换数据库 - - **删除数据库** ```mysql DROP DATABASE [IF EXISTS] db_name; @@ -120,7 +118,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。 - - **显示系统所有数据库** ```mysql SHOW DATABASES; @@ -128,7 +125,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ## 表管理 - **创建数据表** - + ```mysql CREATE TABLE [IF NOT EXISTS] tb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]); ``` @@ -153,7 +150,6 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic 显示当前数据库下的所有数据表信息。说明：可在like中使用通配符进行名称的匹配。 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’_’下划线匹配一个字符。 - - **在线修改显示字符宽度** ```mysql @@ -184,18 +180,18 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ## 超级表STable管理 - **创建超级表** - + ```mysql CREATE TABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); ``` 创建STable, 与创建表的SQL语法相似，但需指定TAGS字段的名称和类型 - + 说明： 1) TAGS 列的数据类型不能是timestamp类型； 2) TAGS 列名不能与其他列名相同； 3) TAGS 列名不能为预留关键字； 4) TAGS 最多允许128个，可以0个，总长度不超过16k个字符 - + - **删除超级表** ```mysql @@ -215,7 +211,7 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ```mysql DESCRIBE stb_name; ``` - + - **超级表增加列** ```mysql @@ -230,11 +226,11 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ## 超级表 STable 中 TAG 管理 - **添加标签** - + ```mysql ALTER TABLE stb_name ADD TAG new_tag_name tag_type; ``` - 为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符. + 为STable增加一个新的标签，并指定新标签的类型。标签总数不能超过128个，总长度不超过16k个字符。 - **删除标签** @@ -265,36 +261,31 @@ TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMic ``` 向表tb_name中插入一条记录 - - **插入一条记录，数据对应到指定的列** ```mysql INSERT INTO tb_name (field1_name, ...) VALUES(field1_value, ...) ``` 向表tb_name中插入一条记录，数据对应到指定的列。SQL语句中没有出现的列，数据库将自动填充为NULL。主键（时间戳）不能为NULL。 - - **插入多条记录** ```mysql INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...)...; ``` 向表tb_name中插入多条记录 - - **按指定的列插入多条记录** ```mysql INSERT INTO tb_name (field1_name, ...) VALUES(field1_value1, ...) (field1_value2, ...) ``` 向表tb_name中按指定的列插入多条记录 - - **向多个表插入多条记录** ```mysql - INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)... + INSERT INTO tb1_name VALUES (field1_value1, ...)(field1_value2, ...)... tb2_name VALUES (field1_value1, ...)(field1_value2, ...)...; ``` 同时向表tb1_name和tb2_name中分别插入多条记录 - - **同时向多个表按列插入多条记录** ```mysql INSERT INTO tb1_name (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) @@ -320,7 +311,7 @@ SELECT select_expr [, select_expr ...] [FILL fill_val] [SLIDING fill_val] [GROUP BY col_list] - [ORDER BY col_list { DESC | ASC }] + [ORDER BY col_list { DESC | ASC }] [SLIMIT limit_val [, SOFFSET offset_val]] [LIMIT limit_val [, OFFSET offset_val]] [>> export_file] @@ -382,7 +373,6 @@ taos> SELECT * FROM meters; Query OK, 9 row(s) in set (0.002022s) ``` - 通配符支持表名前缀，以下两个SQL语句均为返回全部的列： ```mysql SELECT * FROM d1001; @@ -592,11 +582,11 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT COUNT([*|field_name]) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表/超级表中记录行数或某列的非空值个数。 - 返回结果数据类型：长整型INT64。 - 应用字段：应用全部字段。 - 适用于：表、超级表。 - 说明：1）可以使用星号*来替代具体的字段，使用星号(*)返回全部记录数量。2）针对同一表的（不包含NULL值）字段查询结果均相同。3）如果统计对象是具体的列，则返回该列中非NULL值的记录数量。 + 功能说明：统计表/超级表中记录行数或某列的非空值个数。 + 返回结果数据类型：长整型INT64。 + 应用字段：应用全部字段。 + 适用于：表、超级表。 + 说明：1）可以使用星号*来替代具体的字段，使用星号(*)返回全部记录数量。2）针对同一表的（不包含NULL值）字段查询结果均相同。3）如果统计对象是具体的列，则返回该列中非NULL值的记录数量。 示例： ```mysql @@ -613,16 +603,15 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 1 row(s) in set (0.001075s) ``` - - **AVG** ```mysql SELECT AVG(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的平均值。 - 返回结果数据类型：双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool字段。 - 适用于：表、超级表。 - + 功能说明：统计表/超级表中某列的平均值。 + 返回结果数据类型：双精度浮点数Double。 + 应用字段：不能应用在timestamp、binary、nchar、bool字段。 + 适用于：表、超级表。 + 示例： ```mysql taos> SELECT AVG(current), AVG(voltage), AVG(phase) FROM meters; @@ -642,8 +631,8 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT TWA(field_name) FROM tb_name WHERE clause; ``` - 功能说明：时间加权平均函数。统计表/超级表中某列在一段时间内的时间加权平均。 - 返回结果数据类型：双精度浮点数Double。 + 功能说明：时间加权平均函数。统计表/超级表中某列在一段时间内的时间加权平均。 + 返回结果数据类型：双精度浮点数Double。 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 说明：时间加权平均(time weighted average, TWA）查询需要指定查询时间段的 _开始时间_ 和 _结束时间_ 。 适用于：表、超级表。 @@ -652,10 +641,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT SUM(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的和。 - 返回结果数据类型：双精度浮点数Double和长整型INT64。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 适用于：表、超级表。 + 功能说明：统计表/超级表中某列的和。 + 返回结果数据类型：双精度浮点数Double和长整型INT64。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 适用于：表、超级表。 示例： ```mysql @@ -676,9 +665,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT STDDEV(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表中某列的均方差。 - 返回结果数据类型：双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表中某列的均方差。 + 返回结果数据类型：双精度浮点数Double。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 适用于：表。 示例： @@ -694,10 +683,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val是自变量初始值，step_val是自变量的步长值。 - 返回结果数据类型：字符串表达式（斜率, 截距）。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 说明：自变量是时间戳，因变量是该列的值。 + 功能说明：统计表中某列的值是主键（时间戳）的拟合直线方程。start_val是自变量初始值，step_val是自变量的步长值。 + 返回结果数据类型：字符串表达式（斜率, 截距）。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 说明：自变量是时间戳，因变量是该列的值。 适用于：表。 示例： @@ -715,8 +704,8 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最小值。 - 返回结果数据类型：同应用的字段。 + 功能说明：统计表/超级表中某列的值最小值。 + 返回结果数据类型：同应用的字段。 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 示例： @@ -738,8 +727,8 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最大值。 - 返回结果数据类型：同应用的字段。 + 功能说明：统计表/超级表中某列的值最大值。 + 返回结果数据类型：同应用的字段。 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 示例： @@ -757,14 +746,13 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 1 row(s) in set (0.000987s) ``` - - **FIRST** ```mysql SELECT FIRST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最先写入的非NULL值。 - 返回结果数据类型：同应用的字段。 - 应用字段：所有字段。 + 功能说明：统计表/超级表中某列的值最先写入的非NULL值。 + 返回结果数据类型：同应用的字段。 + 应用字段：所有字段。 说明：1）如果要返回各个列的首个（时间戳最小）非NULL值，可以使用FIRST(*)；2) 如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；3) 如果结果集中所有列全部为NULL值，则不返回结果。 示例： @@ -786,9 +774,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT LAST(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最后写入的非NULL值。 - 返回结果数据类型：同应用的字段。 - 应用字段：所有字段。 + 功能说明：统计表/超级表中某列的值最后写入的非NULL值。 + 返回结果数据类型：同应用的字段。 + 应用字段：所有字段。 说明：1）如果要返回各个列的最后（时间戳最大）一个非NULL值，可以使用LAST(*)；2）如果结果集中的某列全部为NULL值，则该列的返回结果也是NULL；如果结果集中所有列全部为NULL值，则不返回结果。 示例： @@ -810,10 +798,10 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT TOP(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。 - 返回结果数据类型：同应用的字段。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 - 说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。 + 功能说明： 统计表/超级表中某列的值最大*k*个非NULL值。若多于k个列值并列最大，则返回时间戳小的。 + 返回结果数据类型：同应用的字段。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。 示例： ```mysql @@ -837,9 +825,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT BOTTOM(field_name, K) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。 - 返回结果数据类型：同应用的字段。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表/超级表中某列的值最小*k*个非NULL值。若多于k个列值并列最小，则返回时间戳小的。 + 返回结果数据类型：同应用的字段。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 说明：1）*k*值取值范围1≤*k*≤100；2）系统同时返回该记录关联的时间戳列。 示例： @@ -863,9 +851,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; ``` - 功能说明：统计表中某列的值百分比分位数。 - 返回结果数据类型： 双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表中某列的值百分比分位数。 + 返回结果数据类型： 双精度浮点数Double。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。 示例： @@ -881,9 +869,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。 - 返回结果数据类型： 双精度浮点数Double。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。 + 返回结果数据类型： 双精度浮点数Double。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 说明：*P*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数 ```mysql taos> SELECT APERCENTILE(current, 20) FROM d1001; @@ -897,9 +885,9 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT LAST_ROW(field_name) FROM { tb_name | stb_name }; ``` - 功能说明：返回表（超级表）的最后一条记录。 - 返回结果数据类型：同应用的字段。 - 应用字段：所有字段。 + 功能说明：返回表（超级表）的最后一条记录。 + 返回结果数据类型：同应用的字段。 + 应用字段：所有字段。 说明：与last函数不同，last_row不支持时间范围限制，强制返回最后一条记录。 示例： @@ -922,11 +910,11 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT DIFF(field_name) FROM tb_name [WHERE clause]; ``` - 功能说明：统计表中某列的值与前一行对应值的差。 - 返回结果数据类型： 同应用字段。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表中某列的值与前一行对应值的差。 + 返回结果数据类型： 同应用字段。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 说明：输出结果行数是范围内总行数减一，第一行没有结果输出。 - + 示例： ```mysql taos> SELECT DIFF(current) FROM d1001; @@ -937,14 +925,13 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 2 row(s) in set (0.001162s) ``` - - **SPREAD** ```mysql SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列的最大值和最小值之差。 - 返回结果数据类型： 双精度浮点数。 - 应用字段：不能应用在binary、nchar、bool类型字段。 + 功能说明：统计表/超级表中某列的最大值和最小值之差。 + 返回结果数据类型： 双精度浮点数。 + 应用字段：不能应用在binary、nchar、bool类型字段。 说明：可用于TIMESTAMP字段，此时表示记录的时间覆盖范围。 示例： @@ -962,15 +949,14 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 1 row(s) in set (0.000836s) ``` - - **四则运算** ```mysql SELECT field_name [+|-|*|/|%][Value|field_name] FROM { tb_name | stb_name } [WHERE clause]; ``` - 功能说明：统计表/超级表中某列或多列间的值加、减、乘、除、取余计算结果。 - 返回结果数据类型：双精度浮点数。 - 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 + 功能说明：统计表/超级表中某列或多列间的值加、减、乘、除、取余计算结果。 + 返回结果数据类型：双精度浮点数。 + 应用字段：不能应用在timestamp、binary、nchar、bool类型字段。 说明：1）支持两列或多列之间进行计算，可使用括号控制计算优先级;2）NULL字段不参与计算，如果参与计算的某行中包含NULL，该行的计算结果为NULL。 ```mysql @@ -987,12 +973,12 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 TDengine支持按时间段进行聚合，可以将表中数据按照时间段进行切割后聚合生成结果，比如温度传感器每秒采集一次数据，但需查询每隔10分钟的温度平均值。这个聚合适合于降维(down sample)操作, 语法如下： ```mysql -SELECT function_list FROM tb_name +SELECT function_list FROM tb_name [WHERE where_condition] INTERVAL (interval [, offset]) [FILL ({NONE | VALUE | PREV | NULL | LINEAR})] -SELECT function_list FROM stb_name +SELECT function_list FROM stb_name [WHERE where_condition] INTERVAL (interval [, offset]) [FILL ({ VALUE | PREV | NULL | LINEAR})] @@ -1000,18 +986,17 @@ SELECT function_list FROM stb_name ``` - 聚合时间段的长度由关键词INTERVAL指定，最短时间间隔10毫秒（10a），并且支持偏移（偏移必须小于间隔）。聚合查询中，能够同时执行的聚合和选择函数仅限于单个输出的函数：count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last，不能使用具有多行输出结果的函数（例如：top、bottom、diff以及四则运算）。 -- WHERE语句可以指定查询的起止时间和其他过滤条件 +- WHERE语句可以指定查询的起止时间和其他过滤条件 - FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种： 1. 不进行填充：NONE(默认填充模式)。 - + 2. VALUE填充：固定值填充，此时需要指定填充的数值。例如：fill(value, 1.23)。 - + 3. NULL填充：使用NULL填充数据。例如：fill(null)。 - - 4. PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。 - -说明： + 4. PREV填充：使用前一个非NULL值填充数据。例如：fill(prev)。 + +说明： 1. 使用FILL语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过1千万条具有插值的结果。 2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了group by语句分组，则返回结果中每个group内不按照时间序列严格单调递增。 @@ -1039,3 +1024,17 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER - 标签最多允许128个，可以0个，标签总长度不超过16k个字符 - SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为1M - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 + +## TAOS SQL其他约定 + +**group by的限制** + +TAOS SQL支持对标签、tbname进行group by操作，也支持普通列进行group by，前提是：仅限一列且该列的唯一值小于10万个。 + +**join操作的限制** + +TAOS SQL支持表之间按主键时间戳来join两张表的列，暂不支持两个表之间聚合后的四则运算。 + +**is not null与不为空的表达式适用范围** + +is not null支持所有类型的列。不为空的表达式为 <>""，仅对非数值类型的列适用。 \ No newline at end of file diff --git a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md index a9a3dee432c57b0078948d15799ac29df0167ae2..95975dba5aeeeee9f42c3bf0b34f48095ea83fa3 100644 --- a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md +++ b/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md @@ -1,6 +1,5 @@ # TDengine 2.0 错误码以及对应的十进制码 - | 状态码 | 模 | 错误码（十六进制） | 错误描述 | 错误码（十进制） | |-----------------------| :---: | :---------: | :------------------------ | ---------------- | |TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647| diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 7f40009e3ca4ac945d273908b6a40d5ee9f7a125..81704c7dbdfeee449b1e9eb4ffb73dac3e03223b 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -12,7 +12,7 @@ Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb) ``` -示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。 +示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。 实际运行的系统往往会根据数据特点的不同，将数据存放在不同的DB里。因此做规划时，也需要考虑。 @@ -35,7 +35,7 @@ TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景 Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable ``` -示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 +示例：1000万台智能电表，每台电表每15分钟采集一次数据，每次采集的数据128字节，那么一年的原始数据量是：10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 用户可以通过参数keep，设置数据在磁盘中的最大保存时长。为进一步减少存储成本，TDengine还提供多级存储，最冷的数据可以存放在最廉价的存储介质上，应用的访问不用做任何调整，只是读取速度降低了。 @@ -80,6 +80,12 @@ TDengine集群的节点数必须大于等于副本数，否则创建表时将报 TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修改配置参数，以满足不同场景的需求。配置文件的缺省位置在/etc/taos目录，可以通过taosd命令行执行参数-c指定配置文件目录。比如taosd -c /home/user来指定配置文件位于/home/user这个目录。 +另外可以使用 “-C” 显示当前服务器配置参数： + +``` +taosd -C +``` + 下面仅仅列出一些重要的配置参数，更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节，而且这些参数的缺省配置都是工作的，一般无需设置。**注意：配置修改后，需要重启*taosd*服务才能生效。** - firstEp: taosd启动时，主动连接的集群中首个dnode的end point, 默认值为localhost:6030。 @@ -97,6 +103,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 - stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。 - queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘，再乘 170 。单位为字节。 +- ratioOfQueryCores: 设置查询线程的最大数量。最小值0 表示只有1个查询线程；最大值2表示最大建立2倍CPU核数的查询线程。默认为1，表示最大和CPU核数相等的查询线程。该值可以为小数，即0.5表示最大建立CPU核数一半的查询线程。 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 @@ -114,7 +121,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - replica：副本个数，取值范围：1-3。单位为个，默认值：1 - precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms -对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： +对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： ``` create database demo days 10 cache 32 blocks 8 replica 3; @@ -141,8 +148,8 @@ ALTER DNODE ``` - dnode_id: 可以通过SQL语句"SHOW DNODES"命令获取 -- config: 要调整的日志参数，在如下列表中取值 - > resetlog 截断旧日志文件，创建一个新日志文件 +- config: 要调整的日志参数，在如下列表中取值 + > resetlog 截断旧日志文件，创建一个新日志文件 > debugFlag < 131 | 135 | 143 > 设置debugFlag为131、135或者143 例如： @@ -150,9 +157,15 @@ ALTER DNODE alter dnode 1 debugFlag 135; ``` -## 客户端配置 +## 客户端配置 + +TDengine系统的前台交互客户端应用程序为taos，以及应用驱动，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。更多taos的使用方法请见Shell命令行程序。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。 -TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同一个配置文件taos.cfg。运行taos时，使用参数-c指定配置文件目录，如taos -c /home/cfg，表示使用/home/cfg/目录下的taos.cfg配置文件中的参数，缺省目录是/etc/taos。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。 +**2.0.10.0 之后版本支持命令行以下参数显示当前客户端参数的配置** + +```bash +taos -C 或 taos --dump-config +``` 客户端配置参数 @@ -163,7 +176,7 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 - locale 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置 - + TDengine为存储中文、日文、韩文等非ASCII编码的宽字符，提供一种专门的字段类型nchar。写入nchar字段的数据将统一采用UCS4-LE格式进行编码并发送到服务器。需要注意的是，编码正确性是客户端来保证。因此，如果用户想要正常使用nchar字段来存储诸如中文、日文、韩文等非ASCII字符，需要正确设置客户端的编码格式。 客户端的输入的字符均采用操作系统当前默认的编码格式，在Linux系统上多为UTF-8，部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中，编码则是CP936。客户端需要确保正确设置自己所使用的字符集，即客户端运行的操作系统当前编码字符集，才能保证nchar中的数据正确转换为UCS4-LE编码格式。 @@ -173,7 +186,7 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 - charset 默认值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置 - + 如果配置文件中不设置charset，在Linux系统中，taos在启动时候，自动读取系统当前的locale信息，并从locale信息中解析提取charset编码格式。如果自动读取locale信息失败，则尝试读取charset配置，如果读取charset配置也失败，则中断启动过程。 在Linux系统中，locale信息包含了字符编码信息，因此正确设置了Linux系统locale以后可以不用再单独设置charset。例如： @@ -215,59 +228,58 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 均是合法的设置东八区时区的格式。 时区的设置对于查询和写入SQL语句中非Unix时间戳的内容（时间戳字符串、关键词now的解析）产生影响。例如： - ``` + ```sql SELECT count(*) FROM table_name WHERE TS<'2019-04-11 12:01:08'; ``` 在东八区，SQL语句等效于 - ``` + ```sql SELECT count(*) FROM table_name WHERE TS<1554955268000; ``` 在UTC时区，SQL语句等效于 - ``` + ```sql SELECT count(*) FROM table_name WHERE TS<1554984068000; ``` 为了避免使用字符串时间格式带来的不确定性，也可以直接使用Unix时间戳。此外，还可以在SQL语句中使用带有时区的时间戳字符串，例如：RFC3339格式的时间戳字符串，2013-04-12T15:52:01.123+08:00或者ISO-8601格式时间戳字符串2013-04-12T15:52:01.123+0800。上述两个字符串转化为Unix时间戳不受系统所在时区的影响。 启动taos时，也可以从命令行指定一个taosd实例的end point，否则就从taos.cfg读取。 - + - maxBinaryDisplayWidth Shell中binary 和 nchar字段的显示宽度上限，超过此限制的部分将被隐藏。默认值：30。可在 shell 中通过命令 set max_binary_display_width nn 动态修改此选项。 - ## 用户管理 系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下： -``` +```sql CREATE USER PASS <'password'>; ``` 创建用户，并指定用户名和密码，密码需要用单引号引起来,单引号为英文半角 -``` +```sql DROP USER ; ``` 删除用户，限root用户使用 -``` +```sql ALTER USER PASS <'password'>; ``` 修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 -``` +```sql ALTER USER PRIVILEGE ; ``` 修改用户权限为：super/write/read，不需要添加单引号 -``` +```mysql SHOW USERS; ``` -显示所有用户 +显示所有用户 **注意：**SQL 语法中，< >表示需要用户输入的部分，但请不要输入< >本身 @@ -316,12 +328,11 @@ taos> DESCRIBE d1001 ``` 那么可以用如下命令导入数据 -``` +```mysql taos> insert into d1001 file '~/data.csv'; Query OK, 9 row(s) affected (0.004763s) ``` - **taosdump工具导入** TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见博客：TDengine DUMP工具使用指南 @@ -334,7 +345,7 @@ TDengine提供了方便的数据库导入导出工具taosdump。用户可以将t 如果用户需要导出一个表或一个STable中的数据，可在shell中运行 -``` +```mysql select * from >> data.csv; ``` @@ -348,37 +359,37 @@ TDengine提供了方便的数据库导出工具taosdump。用户可以根据需 系统管理员可以从CLI查询系统的连接、正在进行的查询、流式计算，并且可以关闭连接、停止正在进行的查询和流式计算。CLI里SQL语法如下： -``` +```mysql SHOW CONNECTIONS; ``` 显示数据库的连接，其中一列显示ip:port, 为连接的IP地址和端口号。 -``` +```mysql KILL CONNECTION ; ``` 强制关闭数据库连接，其中的connection-id是SHOW CONNECTIONS中显示的第一列的数字。 -``` +```mysql SHOW QUERIES; ``` 显示数据查询，其中第一列显示的以冒号隔开的两个数字为query-id，为发起该query应用连接的connection-id和查询次数。 -``` +```mysql KILL QUERY ; ``` 强制关闭数据查询，其中query-id是SHOW QUERIES中显示的 connection-id:query-no字串，如“105:2”，拷贝粘贴即可。 -``` +```mysql SHOW STREAMS; ``` 显示流式计算，其中第一列显示的以冒号隔开的两个数字为stream-id, 为启动该stream应用连接的connection-id和发起stream的次数。 -``` +```mysql KILL STREAM ; ``` @@ -416,8 +427,6 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 - - ## TDengine参数限制与保留关键字 - 数据库名：不能包含“.”以及特殊字符，不能超过32个字符 @@ -436,8 +445,6 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 - 库的个数：仅受节点个数限制 - 单个库上虚拟节点个数：不能超过64个 - - 目前TDengine有将近200个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable名、数据列名及标签列名等。这些关键字列表如下： | 关键字列表 | | | | | @@ -477,5 +484,4 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 | COMP | GE | METRIC | SELECT | VIEW | | CONCAT | GLOB | METRICS | SEMI | WAVG | | CONFIGS | GRANTS | MIN | SET | WHERE | -| CONFLICT | GROUP | | | | - +| CONFLICT | GROUP | | | | \ No newline at end of file diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/webdocs/markdowndocs/advanced features-ch.md index ed02af25f2cd78372f20a2dc074bb0aa30b4e301..cdd9ee81048f0968fb02f036b20e003c66835a4c 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/webdocs/markdowndocs/advanced features-ch.md @@ -60,7 +60,7 @@ create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s 会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句， 并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如： -```shell +```mysql taos> select * from avg_vol; ts | avg_voltage_ | =================================================== @@ -72,14 +72,13 @@ taos> select * from avg_vol; 需要注意，查询时间窗口的最小值是10毫秒，没有时间窗口范围的上限。 - 此外，TDengine还支持用户指定连续查询的起止时间。 如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始； 如果没有输入结束时间，连续查询将永久运行； 如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。 比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。 -```sql +```mysql create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s); ``` diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index c9bfa30830fe7b2f3cd1364b589326255560ad83..47fb8094b7379b9613f204b3d8c773dbff3bb4d3 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -1,4 +1,4 @@ -#数据模型和整体架构 +# 数据模型和整体架构 ## 数据模型 ### 物联网典型场景 @@ -102,7 +102,7 @@ 每一条记录都有设备ID，时间戳，采集的物理量(如上图中的电流、电压、相位），还有与每个设备相关的静态标签（如上述表一中的位置Location和分组groupId）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。 -### 数据特征 +### 数据特征 除时序特征外，仔细研究发现，物联网、车联网、运维监测类数据还具有很多其他明显的特征： 1. 数据高度结构化； @@ -121,7 +121,7 @@ ### 关系型数据库模型 因为采集的数据一般是结构化数据，同时为降低学习门槛，TDengine采用传统的关系型数据库模型管理数据。因此用户需要先创建库，然后创建表，之后才能插入或查询数据。TDengine采用的是结构化存储，而不是NoSQL的key-value存储。 -### 一个数据采集点一张表 +### 一个数据采集点一张表 为充分利用其数据的时序性和其他数据特点，TDengine要求**对每个数据采集点单独建表**（比如有一千万个智能电表，就需创建一千万张表，上述表格中的d1001, d1002, d1003, d1004都需单独建表），用来存储这个采集点所采集的时序数据。这种设计有几大优点： 1. 能保证一个采集点的数据在存储介质上是以块为单位连续存储的。如果读取一个时间段的数据，它能大幅减少随机读取操作，成数量级的提升读取和查询速度。 @@ -150,7 +150,7 @@ TDengine 分布式架构的逻辑结构图如下：
图 1 TDengine架构示意图
一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine应用驱动(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。 -**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)》。 +**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机，可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯，如果不了解FQDN，请看博文《一篇文章说清楚TDengine的FQDN》。 **数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。 @@ -356,7 +356,7 @@ SQL语句的解析和校验工作在客户端完成。解析SQL语句并生成 客户端在获取查询结果的时候，dnode的查询执行队列中的工作线程会等待vnode执行线程执行完成，才能将查询结果返回到请求的客户端。 -### 按时间轴聚合、降采样、插值 +### 按时间轴聚合、降采样、插值 时序数据有别于普通数据的显著特征是每条记录均具有时间戳，因此针对具有时间戳数据在时间轴上进行聚合是不同于普通数据库的重要功能。从这点上来看，与流计算引擎的窗口查询有相似的地方。 diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index db479417c5a22acf26bc7a829083cd21fd7dffb1..f1c275ab0c4c986766fa8d33c71fe65777c90848 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -137,6 +137,16 @@ DROP DNODE "fqdn:port"; 其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号 +**【注意】** + + - 一个数据节点一旦被drop之后，不能重新加入集群。需要将此节点重新部署（清空数据文件夹）。集群在完成drop dnode操作之前，会将该dnode的数据迁移走。 + + - 请注意 drop dnode 和 停止taosd进程是两个不同的概念，不要混淆：因为删除dnode之前要执行迁移数据的操作，因此被删除的dnode必须保持在线状态。待删除操作结束之后，才能停止taosd进程。 + + - 一个数据节点被drop之后，其他节点都会感知到这个dnodeID的删除操作，任何集群中的节点都不会再接收此dnodeID的请求。 + + - dnodeID的是集群自动分配的，不得人工指定。它在生成时递增的，不会重复。 + ### 查看数据节点 执行CLI程序taos,使用root账号登录进TDengine系统，执行： @@ -216,3 +226,5 @@ SHOW MNODES; 如果副本数为偶数，当一个vnode group里一半vnode不工作时，是无法从中选出master的。同理，一半mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 TDengine提供一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/)，在TDengine Arbitrator Linux一节中，选择适合的版本下载并安装。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。如果副本数为奇数，即使配置了arbitrator, 系统也不会去建立连接。 + +关于集群搭建请参考视频教程。 diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md index 5b89761aa4c46e4c223a4b10ae7f7ae41ee537a1..cc6287953ad0561841d7435148ecae07d28b6ca8 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-ch.md @@ -1,36 +1,162 @@ # 连接器 -TDengine提供了丰富的应用程序开发接口，其中包括C/C++、C# 、Java、Python、Go、Node.js、RESTful 等，便于用户快速开发应用。 +TDengine提供了丰富的应用程序开发接口，其中包括C/C++、Java、Python、Go、Node.js、C# 、RESTful 等，便于用户快速开发应用。 ![image-connecotr](../assets/connector.png) -目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下： +目前TDengine的连接器可支持的平台广泛，包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。对照矩阵如下： -| | **CPU** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** | | | -| ---------------------------- | --------- | --------------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ | --------- | --------- | -| | **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | -| **连** **接** **器** | **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | -| **JDBC** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | | -| **Python** | ● | ● | ● | ○ | ● | ● | ● | -- | ● | | -| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | | -| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | | -| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- | | -| **RESTful** | ● | ● | ● | ● | ● | ● | ● | ● | ● | | +| **CPU** | **X64 64bit** | **X64 64bit** | **X64 64bit** | **X86 32bit** | **ARM64** | **ARM32** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** | +| ----------- | --------------- | --------------- | --------------- | --------------- | --------- | --------- | --------------- | ---------------- | -------------- | +| **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | +| **C/C++** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ | +| **JDBC** | ● | ● | ● | ○ | ● | ● | ○ | ○ | ○ | +| **Python** | ● | ● | ● | ○ | ● | ● | ○ | -- | ○ | +| **Go** | ● | ● | ● | ○ | ● | ● | ○ | -- | -- | +| **NodeJs** | ● | ● | ○ | ○ | ● | ● | ○ | -- | -- | +| **C#** | ○ | ● | ● | ○ | ○ | ○ | ○ | -- | -- | +| **RESTful** | ● | ● | ● | ● | ● | ● | ○ | ○ | ○ | -注意：所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。 +其中 ● 表示经过官方测试验证， ○ 表示非官方测试验证。 + +注意： + +* 在没有安装TDengine服务端软件的系统中使用连接器（除RESTful外）访问 TDengine 数据库，需要安装相应版本的客户端安装包来使应用驱动（Linux系统中文件名为libtaos.so，Windows系统中为taos.dll）被安装在系统中，否则会产生无法找到相应库文件的错误。 +* 所有执行 SQL 语句的 API，例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等，以及其它语言中与它们对应的API，每次都只能执行一条 SQL 语句，如果实际参数中包含了多条语句，它们的行为是未定义的。 +* 升级到TDengine到2.0.8.0版本的用户，必须更新JDBC连接TDengine必须升级taos-jdbcdriver到2.0.12及以上。 + +## 安装连接器驱动步骤 + +服务器应该已经安装TDengine服务端安装包。连接器驱动安装步骤如下： + +**Linux** + +**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载** + +* X64硬件环境：TDengine-client-2.x.x.x-Linux-x64.tar.gz + +* ARM64硬件环境：TDengine-client-2.x.x.x-Linux-aarch64.tar.gz + +* ARM32硬件环境：TDengine-client-2.x.x.x-Linux-aarch32.tar.gz + +**2. 解压缩软件包** + +将软件包放置在当前用户可读写的任意目录下，然后执行下面的命令： + +`tar -xzvf TDengine-client-xxxxxxxxx.tar.gz` + +其中xxxxxxx需要替换为实际版本的字符串。 + +**3. 执行安装脚本** + +解压软件包之后，会在解压目录下看到以下文件(目录)： + +​ *install_client.sh*：安装脚本，用于应用驱动程序 +​ *taos.tar.gz*：应用驱动安装包 +​ *driver*：TDengine应用驱动driver +​ *connector*: 各种编程语言连接器（go/grafanaplugin/nodejs/python/JDBC） +​ *examples*: 各种编程语言的示例程序(c/C#/go/JDBC/matlab/python/R) + +运行install_client.sh进行安装 + +**4. 配置taos.cfg** + +编辑taos.cfg文件(默认路径/etc/taos/taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030 + +**提示： 如本机没有部署TDengine服务，仅安装了应用驱动，则taos.cfg中仅需配置firstEP，无需配置FQDN。** + +**Windows x64/x86** + +**1. 从涛思官网（https://www.taosdata.com/cn/all-downloads/）下载 ：** + +* X64硬件环境：TDengine-client-2.X.X.X-Windows-x64.exe + +* X86硬件环境：TDengine-client-2.X.X.X-Windows-x86.exe + +**2. 执行安装程序，按提示选择默认值，完成安装** + +**3. 安装路径** + +默认安装路径为：C:\TDengine，其中包括以下文件(目录)： + +​ *taos.exe*：taos shell命令行程序 + +​ *cfg* : 配置文件目录 +​ *driver*: 应用驱动动态链接库 +​ *examples*: 示例程序 bash/C/C#/go/JDBC/Python/Node.js +​ *include*: 头文件 +​ *log* : 日志文件 +​ *unins000.exe*: 卸载程序 + +**4. 配置taos.cfg** + +编辑taos.cfg文件(默认路径C:\TDengine\cfg\taos.cfg)，将firstEP修改为TDengine服务器的End Point，例如：h1.taos.com:6030 + +**提示：** + +**1. 如利用FQDN连接服务器，必须确认本机网络环境DNS已配置好，或在hosts文件中添加FQDN寻址记录，如编辑C:\Windows\system32\drivers\etc\hosts，添加如下的记录：** **192.168.1.99 h1.taos.com** + +**2．卸载：运行unins000.exe可卸载TDengine应用驱动。** + +**安装验证** + +以上安装和配置完成后，并确认TDengine服务已经正常启动运行，此时可以执行taos客户端进行登录。 + +**Linux环境：** + +在linux shell下直接执行 taos，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下： + +```mysql +$ taos +Welcome to the TDengine shell from Linux, Client Version:2.0.5.0 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. +taos> show databases; +name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB)| blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | +========================================================================================================================================================================================================================= +test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16| 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | +log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1| 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | +Query OK, 2 row(s) in set (0.001198s) +taos> +``` + +**Windows（x64/x86）环境：** + +在cmd下进入到c:\TDengine目录下直接执行 taos.exe，应该就能正常链接到tdegine服务，进入到taos shell界面，示例如下： + +```mysql + C:\TDengine>taos + Welcome to the TDengine shell from Linux, Client Version:2.0.5.0 + Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + taos> show databases; + name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | + =================================================================================================================================================================================================================================================================== + test | 2020-10-14 10:35:48.617 | 10 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | + log | 2020-10-12 09:08:21.651 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | + Query OK, 2 row(s) in set (0.045000s) + taos> +``` ## C/C++ Connector +**C/C++连接器支持的系统有**： + +| **CPU类型** | x64（64bit） | | | ARM64 | ARM32 | +| ------------ | ------------ | -------- | -------- | -------- | ---------- | +| **OS类型** | Linux | Win64 | Win32 | Linux | Linux | +| **支持与否** | **支持** | **支持** | **支持** | **支持** | **开发中** | + C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine头文件 _taos.h_（安装后，位于 _/usr/local/taos/include_）： ```C #include ``` -在编译时需要链接TDengine动态库 _libtaos.so_ （安装后，位于 _/usr/local/taos/driver_，gcc编译时，请加上 -ltaos）。 +注意： -如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 +* 在编译时需要链接TDengine动态库。Linux 为 *libtaos.so* ，安装后，位于 _/usr/local/taos/driver_。Windows为 taos.dll，安装后位于 *C:\TDengine*。 +* 如未特别说明，当API的返回值是整数时，_0_ 代表成功，其它是代表失败原因的错误码，当返回值是指针时， _NULL_ 表示失败。 +使用C/C++连接器的示例代码请参见 https://github.com/taosdata/TDengine/tree/develop/tests/examples/c。 ### 基础API @@ -40,22 +166,18 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine 初始化运行环境。如果应用没有主动调用该API，那么应用在调用`taos_connect`时将自动调用，故应用程序一般无需手动调用该API。 - - `void taos_cleanup()` 清理运行环境，应用退出前应调用此API。 - - `int taos_options(TSDB_OPTION option, const void * arg, ...)` 设置客户端选项，目前只支持时区设置（_TSDB_OPTION_TIMEZONE_）和编码设置（_TSDB_OPTION_LOCALE_）。时区和编码默认为操作系统当前设置。 - - `char *taos_get_client_info()` 获取客户端版本信息。 - - `TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, int port)` 创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含： @@ -65,26 +187,21 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine - pass：密码 - db：数据库名字，如果用户没有提供，也可以正常连接，用户可以通过该连接创建新的数据库，如果用户提供了数据库名字，则说明该数据库用户已经创建好，缺省使用该数据库 - port：端口号 - - 返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。 + 返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。 - `char *taos_get_server_info(TAOS *taos)` 获取服务端版本信息。 - - `int taos_select_db(TAOS *taos, const char *db)` 将当前的缺省数据库设置为`db`。 - - `void taos_close(TAOS *taos)` 关闭连接, 其中`taos`是`taos_connect`函数返回的指针。 - - ### 同步查询API 传统的数据库操作API，都属于同步操作。应用调用API后，一直处于阻塞状态，直到服务器返回结果。TDengine支持如下API： @@ -93,37 +210,30 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine 该API用来执行SQL语句，可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。返回值 NULL 表示失败。 - - `int taos_result_precision(TAOS_RES *res)` 返回结果集时间戳字段的精度，`0` 代表毫秒，`1` 代表微秒，`2` 代表纳秒。 - - `TAOS_ROW taos_fetch_row(TAOS_RES *res)` 按行获取查询结果集中的数据。 - - `int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows)` 批量获取查询结果集中的数据，返回值为获取到的数据的行数。 - - `int taos_num_fields(TAOS_RES *res)` 和 `int taos_field_count(TAOS_RES *res)` 这两个API等价，用于获取查询结果集中的列数。 - - `int* taos_fetch_lengths(TAOS_RES *res)` 获取结果集中每个字段的长度。 返回值是一个数组，其长度为结果集的列数。 - - `int taos_affected_rows(TAOS_RES *res)` 获取被所执行的 SQL 语句影响的行数。 - - `TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)` 获取查询结果集每列数据的属性（数据类型、名字、字节数），与taos_num_fileds配合使用，可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下： @@ -136,30 +246,24 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine } TAOS_FIELD; ``` - - `void taos_stop_query(TAOS_RES *res)` 停止一个查询的执行。 - - `void taos_free_result(TAOS_RES *res)` 释放查询结果集以及相关的资源。查询完成后，务必调用该API释放资源，否则可能导致应用内存泄露。 - - `char *taos_errstr(TAOS_RES *res)` 获取最近一次API调用失败的原因,返回值为字符串。 - - `char *taos_errno(TAOS_RES *res)` 获取最近一次API调用失败的原因，返回值为错误代码。 - **注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。 - ### 异步查询API 同步API之外，TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下，异步API处理数据插入的速度比同步API快2~4倍。异步API采用非阻塞式的调用方式，在系统真正完成某个具体数据库操作前，立即返回。调用的线程可以去处理其他工作，从而可以提升整个应用的性能。异步API在网络延迟严重的情况下，优点尤为突出。 @@ -184,7 +288,6 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine * res：`taos_query_a`回调时返回的结果集 * fp：回调函数。其参数`param`是用户可定义的传递给回调函数的参数结构体；`numOfRows`是获取到的数据的行数（不是整个查询结果集的函数）。 在回调函数中，应用可以通过调用`taos_fetch_row`前向迭代获取批量记录中每一行记录。读完一块内的所有记录后，应用需要在回调函数中继续调用`taos_fetch_rows_a`获取下一批记录进行处理，直到返回的记录数（numOfRows）为零（结果返回完成）或记录数为负值（查询出错）。 - - `void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);` 异步获取一条记录。其中： @@ -194,7 +297,6 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表，并可以同时对每张打开的表进行查询或者插入操作。需要指出的是，**客户端应用必须确保对同一张表的操作完全串行化**，即对同一个表的插入或查询操作未完成时（未返回时），不能够执行第二个插入或查询操作。 - ### 参数绑定API 除了直接调用 `taos_query` 进行查询，TDengine也提供了支持参数绑定的Prepare API，与 MySQL 一样，这些API目前也仅支持用问号`?`来代表待绑定的参数，具体如下： @@ -234,12 +336,11 @@ TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线 - `TAOS_RES* taos_stmt_use_result(TAOS_STMT *stmt)` 获取语句的结果集。结果集的使用方式与非参数化调用时一致，使用完成后，应对此结果集调用 `taos_free_result`以释放资源。 - + - `int taos_stmt_close(TAOS_STMT *stmt)` 执行完毕，释放所有资源。 - ### 连续查询接口 TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时间段，对一张或多张数据库的表(数据流)进行各种实时聚合计算操作。操作简单，仅有打开、关闭流的API。具体如下： @@ -253,14 +354,12 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * stime：是流式计算开始的时间，如果是0，表示从现在开始，如果不为零，表示从指定的时间开始计算（UTC时间从1970/1/1算起的毫秒数） * param：是应用提供的用于回调的一个参数，回调时，提供给应用 * callback: 第二个回调函数，会在连续查询自动停止时被调用。 - - 返回值为NULL，表示创建成功，返回值不为空，表示成功。 + 返回值为NULL，表示创建成功，返回值不为空，表示成功。 - `void taos_close_stream (TAOS_STREAM *tstr)` 关闭数据流，其中提供的参数是taos_open_stream的返回值。用户停止流式计算的时候，务必关闭该数据流。 - ### 数据订阅接口 @@ -285,7 +384,6 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * param：调用 `taos_subscribe`时客户程序提供的附加参数 * code：错误码 - * `TAOS_RES *taos_consume(TAOS_SUB *tsub)` 同步模式下，该函数用来获取订阅的结果。 用户应用程序将其置于一个循环之中。 如两次调用`taos_consume`的间隔小于订阅的轮询周期，API将会阻塞，直到时间间隔超过此周期。 如果数据库有新记录到达，该API将返回该最新的记录，否则返回一个没有记录的空结果集。 如果返回值为 `NULL`，说明系统出错。 异步模式下，用户程序不应调用此API。 @@ -294,13 +392,14 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 取消订阅。 如参数 `keepProgress` 不为0，API会保留订阅的进度信息，后续调用 `taos_subscribe` 时可以基于此进度继续；否则将删除进度信息，后续只能重新开始读取数据。 - ## Python Connector +Python连接器的使用参见视频教程 + ### 安装准备 -* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端 [（Windows TDengine 客户端安装）][4] +* 应用驱动安装请参考安装连接器驱动步骤。 * 已安装python 2.7 or >= 3.4 -* 已安装pip +* 已安装pip 或 pip3 ### Python客户端安装 @@ -312,18 +411,18 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 或 -​ `pip install src/connector/python/linux/python3/` +​ `pip3 install src/connector/python/linux/python3/` #### Windows 在已安装Windows TDengine 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面 ```cmd cd C:\TDengine\connector\python\windows -pip install python2\ +python -m pip install python2\ ``` 或 ```cmd cd C:\TDengine\connector\python\windows -pip install python3\ +python -m pip install python3\ ``` *如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。 @@ -336,7 +435,7 @@ pip install python3\ * 导入TDengine客户端模块 ```python -import taos +import taos ``` * 获取连接并获取游标对象 ```python @@ -348,7 +447,7 @@ c1 = conn.cursor() * 写入数据 ```python import datetime - + # 创建数据库 c1.execute('create database db') c1.execute('use db') @@ -376,7 +475,7 @@ numOfRows = c1.rowcount numOfCols = len(c1.description) for irow in range(numOfRows): print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])) - + # 直接使用cursor 循环拉取查询结果 c1.execute('select * from tb') for data in c1: @@ -402,7 +501,6 @@ for d in data: sub.close() ``` - * 关闭连接 ```python c1.close() @@ -426,7 +524,6 @@ conn.close() 用于生成taos.TDengineConnection的实例。 - ### Python客户端使用示例代码 在tests/examples/python中，我们提供了一个示例Python程序read_example.py，可以参考这个程序来设计用户自己的写入、查询程序。在安装了对应的客户端后，通过import taos引入taos类。主要步骤如下 @@ -439,7 +536,7 @@ conn.close() ## RESTful Connector -为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。 +为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。RESTful连接器的使用参见视频教程。 ### HTTP请求格式 @@ -472,13 +569,13 @@ HTTP请求的BODY里就是一个完整的SQL语句，SQL语句中的数据表应 使用curl通过自定义身份认证方式来发起一个HTTP Request，语法如下： -``` +```bash curl -H 'Authorization: Basic ' -d '' :/rest/sql ``` 或者 -``` +```bash curl -u username:password -d '' :/rest/sql ``` @@ -488,7 +585,7 @@ curl -u username:password -d '' :/rest/sql 返回值为JSON格式，如下: -``` +```json { "status": "succ", "head": ["Time Stamp","current", …], @@ -511,7 +608,7 @@ curl -u username:password -d '' :/rest/sql HTTP请求中需要带有授权码``，用于身份识别。授权码通常由管理员提供，可简单的通过发送`HTTP GET`请求来获取授权码，操作如下： -``` +```bash curl http://:6041/rest/login// ``` @@ -525,13 +622,13 @@ curl http://:6041/rest/login// 获取授权码示例： -``` +```bash curl http://192.168.0.1:6041/rest/login/root/taosdata ``` 返回值： -``` +```json { "status": "succ", "code": 0, @@ -543,12 +640,12 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata - 在demo库里查询表d1001的所有记录： -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sql ``` 返回值： -``` +```json { "status": "succ", "head": ["Time Stamp","current","voltage","phase"], @@ -562,12 +659,12 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 - 创建库demo： -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6041/rest/sql ``` 返回值： -``` +```json { "status": "succ", "head": ["affected_rows"], @@ -582,13 +679,13 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19 HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sqlt ``` 返回值： -``` +```json { "status": "succ", "head": ["column1","column2","column3"], @@ -603,13 +700,13 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 #### 结果集采用UTC时间字符串 HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 -``` +```bash curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6041/rest/sqlutc ``` 返回值： -``` +```json { "status": "succ", "head": ["column1","column2","column3"], @@ -633,31 +730,39 @@ HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间 ## CSharp Connector -在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。 +C#连接器支持的系统有：Linux 64/Windows x64/Windows x86 + +### 安装准备 -#### 安装TDengine客户端 +* 应用驱动安装请参考安装连接器驱动步骤。 +* .NET接口文件TDengineDrivercs.cs和参考程序示例TDengineTest.cs均位于Windows客户端install_directory/examples/C#目录下。 +* 在Windows系统上，C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作，后续版本将提供ORM（dapper）框架驱动。 -C#连接器需要使用`libtaos.so`和`taos.h`。因此，在使用C#连接器之前，需在程序运行的Windows环境安装TDengine的Windows客户端，以便获得相关驱动文件。 +### 安装验证 -安装完成后，在文件夹`C:/TDengine/examples/C#`中，将会看到两个文件 +运行install_directory/examples/C#/C#Checker/C#Checker.exe -- TDengineDriver.cs 调用taos.dll文件的Native C方法 -- TDengineTest.cs 参考程序示例 +```cmd +cd {install_directory}/examples/C#/C#Checker +csc /optimize *.cs +C#Checker.exe -h +``` -在文件夹`C:\Windows\System32`，将会看到`taos.dll`文件 +### C#连接器的使用 -#### 使用方法 +在Windows系统上，.NET应用程序可以使用TDengine的.NET接口来执行所有数据库的操作。使用.NET接口的步骤如下所示： -- 将C#接口文件TDengineDriver.cs加入到应用程序所在.NET项目中 -- 参考TDengineTest.cs来定义数据库连接参数，及执行数据插入、查询等操作的方法 -- 因为C#接口需要用到`taos.dll`文件，用户可以将`taos.dll`文件加入.NET解决方案中 +1. 将.NET接口文件TDengineDrivercs.cs加入到应用程序所在.NET项目中。 +2. 用户可以参考TDengineTest.cs来定义数据库连接参数，以及如何执行数据插入、查询等操作； -#### 注意事项 +此.NET接口需要用到taos.dll文件，所以在执行应用程序前，拷贝Windows客户端install_directory/driver目录中的taos.dll文件到.NET项目最后生成.exe可执行文件所在文件夹。之后运行exe文件，即可访问TDengine数据库并做插入、查询等操作。 -- `taos.dll`文件使用x64平台编译，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请均选择“x64”。 -- 此.NET接口目前已经在Visual Studio 2013/2015/2017中验证过，其它VS版本尚待验证。 +**注意：** -#### 第三方驱动 +1. TDengine V2.0.3.0之后同时支持32位和64位Windows系统，所以.NET项目在生成.exe文件时，“解决方案”/“项目”的“平台”请选择对应的“X86” 或“x64”。 +2. 此.NET接口目前已经在Visual Studio 2015/2017中验证过，其它VS版本尚待验证。 + +### 第三方驱动 Maikebing.Data.Taos是一个TDengine的ADO.Net提供器，支持linux，windows。该开发包由热心贡献者`麦壳饼@@maikebing`提供，具体请参考 @@ -668,10 +773,15 @@ https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos https://www.taosdata.com/blog/2020/11/02/1901.html ``` - ## Go Connector -TDengine提供了GO驱动程序`taosSql`. `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go` +### 安装准备 + +* 应用驱动安装请参考安装连接器驱动步骤。 + +TDengine提供了GO驱动程序`taosSql`。 `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`。 + +使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go 以及视频教程。 ```Go import ( @@ -679,16 +789,16 @@ import ( _ "github.com/taosdata/driver-go/taosSql" ) ``` -**建议Go版本是1.13或以上，并开启模块支持：** +**建议使用Go版本1.13或以上，并开启模块支持：** -``` +```bash go env -w GO111MODULE=on go env -w GOPROXY=https://goproxy.io,direct ``` ### 常用API -- sql.Open(DRIVER_NAME string, dataSourceName string) *DB` +- `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` 该API用来打开DB，返回一个类型为*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine @@ -720,50 +830,90 @@ go env -w GOPROXY=https://goproxy.io,direct ## Node.js Connector -TDengine 同时也提供了node.js 的连接器。用户可以通过[npm](https://www.npmjs.com/)来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。[具体安装步骤如下](https://github.com/taosdata/tdengine/tree/master/src/connector/nodejs)： +Node.js连接器支持的系统有： -首先，通过[npm](https://www.npmjs.com/)安装node.js 连接器. +| **CPU类型** | x64（64bit） | | | aarch64 | aarch32 | +| ------------ | ------------ | -------- | -------- | -------- | -------- | +| **OS类型** | Linux | Win64 | Win32 | Linux | Linux | +| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** | -```cmd +Node.js连接器的使用参见视频教程 + +### 安装准备 + +* 应用驱动安装请参考安装连接器驱动步骤。 + +### 安装Node.js连接器 + +用户可以通过npm来进行安装，也可以通过源代码*src/connector/nodejs/* 来进行安装。具体安装步骤如下： + +首先，通过npm安装node.js 连接器. + +```bash npm install td2.0-connector ``` 我们建议用户使用npm 安装node.js连接器。如果您没有安装npm, 可以将*src/connector/nodejs/*拷贝到您的nodejs 项目目录下 -我们使用[node-gyp](https://github.com/nodejs/node-gyp)和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件： +我们使用node-gyp和TDengine服务端进行交互。安装node.js 连接器之前，还需安装以下软件： ### Linux - `python` (建议`v2.7` , `v3.x.x` 目前还不支持) - `node` 必须采用v10.x版本，其他版本存在包兼容性的问题。 - `make` -- c语言编译器比如[GCC](https://gcc.gnu.org) +- c语言编译器比如GCC ### Windows #### 安装方法1 -使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具 +使用微软的windows-build-tools在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具 #### 安装方法2 手动安装以下工具: -- 安装Visual Studio相关：[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) -- 安装 [Python 2.7](https://www.python.org/downloads/) (`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` +- 安装Visual Studio相关：Visual Studio 2017 Community +- 安装 Python 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` - 进入`cmd`命令行界面, `npm config set msvs_version 2017` -如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules) +如果以上步骤不能成功执行, 可以参考微软的node.js用户手册Microsoft's Node.js Guidelines for Windows 如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64". -### 使用方法 +### 示例程序 + +示例程序源码位于install_directory/examples/nodejs，有： + +Node-example.js node.js示例源程序 +Node-example-raw.js + +### 安装验证 + +在安装好TDengine客户端后，使用nodejsChecker.js程序能够验证当前环境是否支持nodejs方式访问Tdengine。 + +验证方法： + +1. 新建安装验证目录，例如：~/tdengine-test，拷贝github上nodejsChecker.js源程序。下载地址：（https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js）。 + +2. 在命令中执行以下命令： + +```bash +npm init -y +npm install td2.0-connector +node nodejsChecker.js host=localhost +``` + +3. 执行以上步骤后，在命令行会输出nodejs连接Tdengine实例，并执行简答插入和查询的结果。 + +### Node.js连接器的使用 (http://docs.taosdata.com/node) -以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考[该文档](http://docs.taosdata.com/node) +以下是node.js 连接器的一些基本使用方法，详细的使用方法可参考该文档 -#### 连接 +#### 建立连接 -使用node.js连接器时，必须先require ```td2.0-connector```，然后使用 ```taos.connect``` 函数。```taos.connect``` 函数必须提供的参数是```host```，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化```cursor``` 来和TDengine服务端通信 +使用node.js连接器时，必须先require `td2.0-connector`，然后使用 `taos.connect` 函数。`taos.connect` 函数必须提供的参数是`host`，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化`cursor` 来和TDengine服务端通信 ```javascript const taos = require('td2.0-connector'); @@ -777,15 +927,35 @@ var cursor = conn.cursor(); // Initializing a new cursor conn.close(); ``` +#### 执行SQL和插入数据 + +对于DDL语句（例如create database、create table、use等），可以使用cursor的execute方法。代码如下： + +```js +cursor.execute('create database if not exists test;') +``` + +以上代码创建了一个名称为test的数据库。对于DDL语句，一般没有返回值，cursor的execute返回值为0。 + +对于Insert语句，代码如下： + +```js +var affectRows = cursor.execute('insert into test.weather values(now, 22.3, 34);') +``` + +execute方法的返回值为该语句影响的行数，上面的sql向test库的weather表中，插入了一条数据，则返回值affectRows为1。 + +TDengine目前还不支持update和delete语句。 + #### 查询 -可通过 ```cursor.query``` 函数来查询数据库。 +可通过 `cursor.query` 函数来查询数据库。 ```javascript var query = cursor.query('show databases;') ``` -查询的结果可以通过 ```query.execute()``` 函数获取并打印出来 +查询的结果可以通过 `query.execute()` 函数获取并打印出来 ```javascript var promise = query.execute(); @@ -793,7 +963,7 @@ promise.then(function(result) { result.pretty(); }); ``` -格式化查询语句还可以使用```query```的```bind```方法。如下面的示例：```query```会自动将提供的数值填入查询语句的```?```里。 +格式化查询语句还可以使用`query`的`bind`方法。如下面的示例：`query`会自动将提供的数值填入查询语句的`?`里。 ```javascript var query = cursor.query('select * from meterinfo.meters where ts <= ? and areaid = ?;').bind(new Date(), 5); @@ -801,8 +971,7 @@ query.execute().then(function(result) { result.pretty(); }) ``` -如果在```query```语句里提供第二个参数并设为```true```也可以立即获取查询结果。如下： - +如果在`query`语句里提供第二个参数并设为`true`也可以立即获取查询结果。如下： ```javascript var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true) @@ -823,12 +992,7 @@ promise2.then(function(result) { }) ``` - ### 示例 -[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例 - -[这里](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`. - - -[4]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client +这里提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例 +这里同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`. diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md index f17c74bdf3b0d5f8fc6837b0769fbad6e620e442..7ba573d2e44bb3848d13a2f6b7cb81038843a291 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md @@ -1,17 +1,23 @@ # Java Connector -Java连接器支持的系统有： Linux 64/Windows x64/Windows x86。 +Java连接器支持的系统有： +| **CPU类型** | x64（64bit） | | | ARM64 | ARM32 | +| ------------ | ------------ | -------- | -------- | -------- | -------- | +| **OS类型** | Linux | Win64 | Win32 | Linux | Linux | +| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** | + +Java连接器的使用请参见视频教程。 TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。 由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 -* libtaos.so +* libtaos.so 在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 - + * taos.dll 在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 - + > 注意：在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点： @@ -21,7 +27,6 @@ TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一 * 目前不支持表间的 union 操作。 * 目前不支持嵌套查询(nested query)，对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet。 - ## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 | taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | @@ -70,7 +75,6 @@ maven 项目中使用如下 pom.xml 配置即可： 下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。 - ## 使用说明 ### 获取连接 @@ -212,7 +216,6 @@ while(resultSet.next()){ ``` > 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。 - ### 订阅 #### 创建 @@ -227,7 +230,7 @@ TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from met * sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据 * restart：如果订阅已经存在，是重新开始，还是继续之前的订阅 -如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic' 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。 +如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。 #### 消费数据 @@ -255,7 +258,6 @@ sub.close(true); `close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。 - ### 关闭资源 ```java @@ -263,7 +265,7 @@ resultSet.close(); stmt.close(); conn.close(); ``` -> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。 +> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。 ## 与连接池使用 @@ -289,18 +291,18 @@ conn.close(); config.setMinimumIdle(3); //minimum number of idle connection config.setMaximumPoolSize(10); //maximum number of connection in the pool config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool - config.setIdleTimeout(60000); // max idle time for recycle idle connection + config.setIdleTimeout(60000); // max idle time for recycle idle connection config.setConnectionTestQuery("describe log.dn"); //validation query config.setValidationTimeout(3000); //validation query timeout HikariDataSource ds = new HikariDataSource(config); //create datasource - + Connection connection = ds.getConnection(); // get connection Statement statement = connection.createStatement(); // get statement - - //query or insert + + //query or insert // ... - + connection.close(); // put back to conneciton pool } ``` @@ -342,7 +344,7 @@ public static void main(String[] args) throws Exception { properties.put("testWhileIdle","true"); // test connection while idle properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true properties.put("testOnReturn","false"); // don't need while testWhileIdle is true - + //create druid datasource DataSource ds = DruidDataSourceFactory.createDataSource(properties); Connection connection = ds.getConnection(); // get connection @@ -376,15 +378,15 @@ Query OK, 1 row(s) in set (0.000141s) ## 常见问题 * java.lang.UnsatisfiedLinkError: no taos in java.library.path - + **原因**：程序没有找到依赖的本地函数库 taos。 - + **解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 ` ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 - + * java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform - + **原因**：目前 TDengine 只支持 64 位 JDK。 - + **解决方法**：重新安装 64 位 JDK。 * 其它问题请参考 [Issues][7] @@ -399,7 +401,7 @@ Query OK, 1 row(s) in set (0.000141s) [8]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver [9]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver [10]: https://maven.aliyun.com/mvn/search -[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate +[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate [12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo [13]: https://www.taosdata.com/cn/documentation20/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE [14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index d89cdd4c927ec7224d915bb1b3bc184895155d43..a085e6159a37aec1699e3e5100ddeec6d71f4fb6 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -27,43 +27,40 @@ * 云服务器：检查云服务器的安全组是否打开TCP/UDP 端口6030-6042的访问权限 * 本地虚拟机：检查网络能否ping通，尽量避免使用`localhost` 作为hostname * 公司服务器：如果为NAT网络环境，请务必检查服务器能否将消息返回值客户端 - + 2. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用 3. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd* -4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 +4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)）,FQDN配置参考：一篇文章说清楚TDengine的FQDN。 5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的 -7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 +7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) 9. 如果仍不能排除连接故障 - + * Linux 系统请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` 检查服务器侧TCP端口连接是否工作：`nc -l {port}` 检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}` - - * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问 - -10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。 + * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问 +10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：TDengine 内嵌网络检测工具使用指南。 ## 6. 遇到错误“Unexpected generic error in RPC”或者"TDengine Error: Unable to resolve FQDN"， 我怎么办？ 产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： -1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 +1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考：一篇文章说清楚TDengine的FQDN。 2. 如果网络配置有DNS server, 请检查是否正常工作 3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件，查看该FQDN是否配置，并是否有正确的IP地址。 4. 如果网络配置OK，从客户端所在机器，你需要能Ping该连接的FQDN，否则客户端是无法连接服务器的 - ## 7. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。 @@ -86,7 +83,7 @@ TDengine还没有一组专用的validation queries。然而建议你使用系统 ## 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ -windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： +Windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： ```JAVA Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); @@ -94,7 +91,7 @@ properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); Connection = DriverManager.getConnection(url, properties); ``` ## 12.TDengine GO windows驱动的如何编译？ -请看为此问题撰写的技术博客 +请看为此问题撰写的技术博客 ## 13.JDBC报错： the excuted SQL is not a DML or a DDL？ 请更新至最新的JDBC驱动 @@ -105,18 +102,14 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 14. taos connect failed, reason: invalid timestamp +## 14. taos connect failed, reason: invalid timestamp 常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。 - - ## 15. 表名显示不全 由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。 - - ## 16. 如何进行数据迁移？ TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A移动机器B时，注意如下两件事： @@ -125,13 +118,11 @@ TDengine是根据hostname唯一标志一台机器的，在数据文件从机器A - 2.0.7.0 及以后的版本，到/var/lib/taos/dnode下，修复dnodeEps.json的dnodeId对应的FQDN，重启。确保机器内所有机器的此文件是完全相同的。 - 1.x 和 2.x 版本的存储结构不兼容，需要使用迁移工具或者自己开发应用导出导入数据。 - - ## 17. 怎么报告问题？ -如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: +如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包： 1. /var/log/taos -2. /etc/taos +2. /etc/taos 附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md index b303d0d5fbb9e2d7435efb3e509b46bad4c8910b..3fa48c1f508bedf7b4000ffe9f7ef8c96e42d606 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/webdocs/markdowndocs/insert-ch.md @@ -28,16 +28,16 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, - 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天，那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2，那么无法写入比当前时间还晚2天的数据。 ## Prometheus直接写入 -[Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 +Prometheus作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具Bailongma，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文用Docker容器快速搭建一个Devops监控Demo即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 ### 从源代码编译blm_prometheus -用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件： +用户需要从github下载Bailongma的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件： - Linux操作系统的服务器 - 安装好Golang, 1.10版本以上 - 对应的TDengine版本。因为用到了TDengine的客户端动态链接库，因此需要安装好和服务端相同版本的TDengine程序；比如服务端版本是TDengine 2.0.0, 则在bailongma所在的linux服务器（可以与TDengine在同一台服务器，或者不同服务器） Bailongma项目中有一个文件夹blm_prometheus，存放了prometheus的写入API程序。编译过程如下： -``` +```bash cd blm_prometheus go build ``` @@ -45,10 +45,10 @@ go build 一切正常的情况下，就会在对应的目录下生成一个blm_prometheus的可执行程序。 ### 安装Prometheus -通过Prometheus的官网下载安装。[下载地址](https://prometheus.io/download/) +通过Prometheus的官网下载安装。下载地址 ### 配置Prometheus -参考Prometheus的[配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/),在Prometheus的配置文件中的部分，增加以下配置 +参考Prometheus的配置文档，在Prometheus的配置文件中的部分，增加以下配置 - url: bailongma API服务提供的URL, 参考下面的blm_prometheus启动示例章节 @@ -60,7 +60,7 @@ blm_prometheus程序有以下选项，在启动blm_prometheus程序时可以通 --tdengine-name 如果TDengine安装在一台具备域名的服务器上，也可以通过配置TDengine的域名来访问TDengine。在K8S环境下，可以配置成TDengine所运行的service name ---batch-size +--batch-size blm_prometheus会将收到的prometheus的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。 --dbname @@ -79,7 +79,7 @@ blm_prometheus对prometheus提供服务的端口号。 ### 启动示例 通过以下命令启动一个blm_prometheus的API服务 -``` +```bash ./blm_prometheus -port 8088 ``` 假设blm_prometheus所在服务器的IP地址为"10.1.2.3"，则在prometheus的配置文件中部分增加url为 @@ -107,16 +107,16 @@ prometheus产生的数据格式如下： } ``` 其中，apiserver_request_latencies_bucket为prometheus采集的时序数据的名称，后面{}中的为该时序数据的标签。blm_prometheus会以时序数据的名称在TDengine中自动创建一个超级表，并将{}中的标签转换成TDengine的tag值，Timestamp作为时间戳，value作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。 -``` +```mysql use prometheus; select * from apiserver_request_latencies_bucket; ``` ## Telegraf直接写入 -[Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)是一流行的IT运维数据采集开源工具，TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 +是一流行的IT运维数据采集开源工具，TDengine提供一个小工具Bailongma，只需在Telegraf做简单配置，无需任何代码，就可将Telegraf采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文用Docker容器快速搭建一个Devops监控Demo即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 ### 从源代码编译blm_telegraf -用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件： +用户需要从github下载Bailongma的源码，使用Golang语言编译器编译生成可执行文件。在开始编译前，需要准备好以下条件： - Linux操作系统的服务器 - 安装好Golang, 1.10版本以上 @@ -124,7 +124,7 @@ select * from apiserver_request_latencies_bucket; Bailongma项目中有一个文件夹blm_telegraf，存放了Telegraf的写入API程序。编译过程如下： -``` +```bash cd blm_telegraf go build ``` @@ -137,7 +137,7 @@ go build ### 配置Telegraf 修改Telegraf配置文件/etc/telegraf/telegraf.conf中与TDengine有关的配置项。 -在output plugins部分，增加[[outputs.http]]配置项： +在output plugins部分，增加[[outputs.http]]配置项： - url： bailongma API服务提供的URL, 参考下面的启动示例章节 - data_format: "json" @@ -148,16 +148,16 @@ go build - hostname: 区分不同采集设备的机器名称，需确保其唯一性 - metric_batch_size: 100，允许Telegraf每批次写入记录最大数量，增大其数量可以降低Telegraf的请求发送频率。 -关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息，请参考Telegraf官方的[文档](https://docs.influxdata.com/telegraf/v1.11/)。 +关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息，请参考Telegraf官方的文档。 ### 启动blm_telegraf程序 blm_telegraf程序有以下选项，在启动blm_telegraf程序时可以通过设定这些选项来设定blm_telegraf的配置。 ```sh ---host +--host TDengine服务端的IP地址，缺省值为空 ---batch-size +--batch-size blm_telegraf会将收到的telegraf的数据拼装成TDengine的写入请求，这个参数控制一次发给TDengine的写入请求中携带的数据条数。 --dbname @@ -175,7 +175,7 @@ blm_telegraf对telegraf提供服务的端口号。 ### 启动示例 通过以下命令启动一个blm_telegraf的API服务 -``` +```bash ./blm_telegraf -host 127.0.0.1 -port 8089 ``` @@ -213,20 +213,17 @@ telegraf产生的数据格式如下： 其中，name字段为telegraf采集的时序数据的名称，tags字段为该时序数据的标签。blm_telegraf会以时序数据的名称在TDengine中自动创建一个超级表，并将tags字段中的标签转换成TDengine的tag值，Timestamp作为时间戳，fields字段中的值作为该时序数据的值。因此在TDengine的客户端中，可以通过以下指令查到这个数据是否成功写入。 -``` +```mysql use telegraf; select * from cpu; ``` - - MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。 ## EMQ Broker 直接写入 -[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 +EMQ是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考EMQ 官方文档。 ## HiveMQ Broker 直接写入 -[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。 - +HiveMQ 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 HiveMQ extension - TDengine 说明文档。 diff --git a/documentation20/webdocs/markdowndocs/replica-ch.md b/documentation20/webdocs/markdowndocs/replica-ch.md index 1d80174455913a5de55678151811218e6f3d749c..4d714fb5502534521e47d761e24af67b31248f51 100644 --- a/documentation20/webdocs/markdowndocs/replica-ch.md +++ b/documentation20/webdocs/markdowndocs/replica-ch.md @@ -30,7 +30,7 @@ TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储 - master： 具有最新的数据，容许客户端往里写入数据，一个虚拟节点组，至多一个master. - slave：与master是同步的，但不容许客户端往里写入数据，根据配置，可以容许客户端对其进行查询。 -- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务 +- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务。 - offline: 由于宕机或网络原因，无法访问到某虚拟节点时，其他虚拟节点将该虚拟节点标为离线。但请注意，该虚拟节点本身的状态可能是unsynced或其他，但不会是离线。 **Quorum:** @@ -83,10 +83,10 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性 如果一个虚拟节点(vnode A)检测到与同一虚拟节点组内另外一虚拟节点（vnode B）的连接中断，vnode A将立即把vnode B的role设置为offline。无论是接收到另外一虚拟节点发来的status消息，还是检测与另外一虚拟节点的连接中断，该虚拟节点都将进入状态处理流程。状态处理流程的规则如下： 1. 如果检测到在线的节点数没有超过一半，则将自己的状态设置为unsynced. -2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程 +2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程。 3. 如果master不存在，则会检查自己保存的各虚拟节点的状态信息与从另一节点接收到的是否一致，如果一致，说明节点组里状态已经稳定一致，则会触发选举流程。如果不一致，说明状态还没趋于一致，即使master不存在，也不进行选主。由于要求状态信息一致才进行选举，每个虚拟节点根据同样的信息，会选出同一个虚拟节点做master，无需投票表决。 4. 自己的状态是根据规则自己决定并修改的，并不需要其他节点同意，包括成为master。一个节点无权修改其他节点的状态。 -5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version). +5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version)。 具体的流程图如下： @@ -124,7 +124,7 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性 如果一虚拟节点(vnode B) 处于unsynced状态，master存在（vnode A)，而且其版本号比master的低，它将立即启动数据恢复流程。在理解恢复流程时，需要澄清几个关于文件的概念和处理规则。 -1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mnode里的文件，数量是固定的，对应于acct, user, db, table等文件。 +1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mode里的文件，数量是固定的，对应于acct, user, db, table等文件。 2. 任何一个数据文件(file)有名字、大小，还有一个magic number。只有文件名、大小与magic number一致时，两个文件才判断是一样的，无需同步。Magic number可以是checksum, 也可以是简单的文件大小。怎么计算magic，换句话说，如何检测数据文件是否有效，完全由应用决定。 3. 文件名的处理有点复杂，因为每台服务器的路径可能不一致。比如node A的TDengine的数据文件存放在 /etc/taos目录下，而node B的数据存放在 /home/jhtao目录下。因此同步模块需要应用在启动一个同步实例时提供一个path，这样两台服务器的绝对路径可以不一样，但仍然可以做对比，做同步。 4. 当sync模块调用回调函数getFileInfo获得数据文件信息时，有如下的规则 @@ -212,10 +212,10 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系 相同之处： -- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程 -- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的 +- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程。 +- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的。 - 数据转发流程完全一样，Master(leader)需要等待回复确认。 -- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步 +- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步。 不同之处： @@ -226,7 +226,7 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系 ## Meta Data的数据复制 -TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析 +TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析。 TDengine里Meta Data包括以下： diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 8a68d02dfd232e557d0877e738aa8aa33a02467f..73fa915abdde8e0940f142fc56124f66e32a6d25 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -29,16 +29,20 @@ # number of threads per CPU core # numOfThreadsPerCore 1.0 -# the proportion of total threads responsible for query -# ratioOfQueryThreads 0.5 +# the proportion of total CPU cores available for query processing +# 2.0: the query threads will be set to double of the CPU cores. +# 1.0: all CPU cores are available for query processing [default]. +# 0.5: only half of the CPU cores are available for query. +# 0.0: only one core available. +# tsRatioOfQueryCores 1.0 # number of management nodes in the system # numOfMnodes 3 -# enable/disable backuping vnode directory when removing dnode +# enable/disable backuping vnode directory when removing vnode # vnodeBak 1 -# if report installation / use information +# enable/disable installation / usage report # telemetryReporting 1 # enable/disable load balancing @@ -77,7 +81,7 @@ # minimum time window, milli-second # minIntervalTime 10 -# maximum delay before launching a stream compution, milli-second +# maximum delay before launching a stream computation, milli-second # maxStreamCompDelay 20000 # maximum delay before launching a stream computation for the first time, milli-second @@ -152,7 +156,7 @@ # max number of connections allowed in dnode # maxShellConns 5000 -# max numerber of connections allowed in client +# max number of connections allowed in client # maxConnections 5000 # stop writing logs when the disk size of the log folder is less than this value @@ -183,7 +187,7 @@ # restfulRowLimit 10240 # The following parameter is used to limit the maximum number of lines in log files. -# max number of rows per log filters +# max number of lines per log filters # numOfLogLines 10000000 # enable/disable async log @@ -195,7 +199,9 @@ # The following parameters are used for debug purpose only. # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log. +# 131: output warning and error +# 135: output debug, warning and error +# 143: output trace, debug, warning and error to log # 199: output debug, warning and error to both screen and file # 207: output trace, debug, warning and error to both screen and file @@ -227,10 +233,10 @@ # cDebugFlag 131 # debug flag for JNI -# jniDebugflag 131 +# jniDebugFlag 131 # debug flag for storage -# uDebugflag 131 +# uDebugFlag 131 # debug flag for http server # httpDebugFlag 131 @@ -239,12 +245,12 @@ # monDebugFlag 131 # debug flag for query -# qDebugflag 131 +# qDebugFlag 131 # debug flag for vnode -# vDebugflag 131 +# vDebugFlag 131 -# debug flag for http server +# debug flag for TSDB # tsdbDebugFlag 131 # debug flag for continue query @@ -265,5 +271,5 @@ # enable/disable stream (continuous query) # stream 1 -# only 50% CPU resources will be used in query processing -# halfCoresForQuery 0 +# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode +# retrieveBlockingModel 0 diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index d24502a1cb8e69ddaf3989a89e51cc07dfb55f00..3d57ece2adf4e8efd3338f52787d5d9424016e78 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -26,6 +26,7 @@ else ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index edc7de96923266729d1985cb43abfa3003f9178e..431093be950e2c2bb199ed294b57641290fe6221 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/taosdemox ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 8c23ab802d54088dafc4b4c6a5d1b6241c881f2e..6f012aa80eac67ba42e7ca561c95ef6d905c2100 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosdemox %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include @@ -135,7 +136,8 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - #${csudo} rm -f ${bin_link_dir}/taosdump || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index d6dccf7045a02a3019d95d73685a3ca27bc5205f..338abcc6a05603e3ce313fc615a6240754e5315a 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -9,6 +9,9 @@ set -e verMode=edge pagMode=full +iplist="" +serverFqdn="" + # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory @@ -172,6 +175,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : @@ -183,6 +187,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : @@ -227,6 +232,157 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname -f) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -247,7 +403,9 @@ function install_config() { if [ "$interactiveFqdn" == "no" ]; then return 0 - fi + fi + + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" @@ -446,6 +604,9 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + #${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/setDelay.sh' >> ${taosd_service_config}" + #${csudo} bash -c "echo 'ExecStartPost=/usr/local/taos/bin/resetDelay.sh' >> ${taosd_service_config}" + #${csudo} bash -c "echo 'ExecStopPost=/usr/local/taos/bin/resetDelay.sh' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" @@ -454,6 +615,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo 'Restart=always' >> ${taosd_service_config}" ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${taosd_service_config}" ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${taosd_service_config}" + #${csudo} bash -c "echo 'StartLimitIntervalSec=60s' >> ${taosd_service_config}" ${csudo} bash -c "echo >> ${taosd_service_config}" ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" @@ -634,9 +796,9 @@ function update_TDengine() { fi if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" else - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" + echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos -h $serverFqdn${NC} in shell${NC}" fi echo @@ -655,8 +817,8 @@ function update_TDengine() { function install_TDengine() { # Start to install if [ ! -e taos.tar.gz ]; then - echo "File taos.tar.gz does not exist" - exit 1 + echo "File taos.tar.gz does not exist" + exit 1 fi tar -zxf taos.tar.gz @@ -702,41 +864,54 @@ function install_TDengine() { echo echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}" + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} systemctl start taosd${NC}" elif ((${service_mod}==1)); then - echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" + echo -e "${GREEN_DARK}To start TDengine ${NC}: ${csudo} service taosd start${NC}" else - echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" + echo -e "${GREEN_DARK}To start TDengine ${NC}: taosd${NC}" fi - if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" - else - echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" - fi + #if [ ${openresty_work} = 'true' ]; then + # echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + #else + # echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" + #fi if [ ! -z "$firstEp" ]; then - echo - echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp${GREEN_DARK} to login into cluster, then${NC}" - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" + echo fi + echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo else # Only install client install_bin install_config - echo echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" fi touch ~/.taos_history - rm -rf $(tar -tf taos.tar.gz) } ## ==============================Main program starts from here============================ +serverFqdn=$(hostname -f) if [ "$verType" == "server" ]; then # Install server and client if [ -x ${bin_dir}/taosd ]; then diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 0467300953ca34d1f9b24ba9999ccd156a0d8d09..dd116e9bfb0472b661321e9af786521300ed2f74 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -86,6 +86,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : fi ${csudo} rm -f ${bin_link_dir}/rmtaos || : @@ -97,6 +98,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 26977e12f4b7ee780948399605f28603817f1aa2..04fd23d5abb2874d0b3a68b937be529bd2c91a59 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -85,8 +85,9 @@ function install_bin() { # Remove links ${csudo} rm -f ${bin_link_dir}/power || : if [ "$osType" != "Darwin" ]; then - ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdump || : + ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdemox || : + ${csudo} rm -f ${bin_link_dir}/powerdump || : fi ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -97,6 +98,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : + [ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || : [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || : fi [ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || : diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index 5929b52afca59f5e4364aa294fe7cefec5f6b828..28788ceb74ac553da4f72a405085cfe264e47fa2 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -9,6 +9,8 @@ set -e verMode=edge pagMode=full +iplist="" +serverFqdn="" # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory @@ -172,7 +174,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerd || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdump || : + ${csudo} rm -f ${bin_link_dir}/powerdemox || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -183,7 +185,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : [ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd || : [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : - [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || : + [ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || : [ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : @@ -227,6 +229,157 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname -f) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -248,6 +401,8 @@ function install_config() { if [ "$interactiveFqdn" == "no" ]; then return 0 fi + + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" @@ -611,9 +766,9 @@ function update_PowerDB() { fi if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" else - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" + echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell${NC}" fi echo @@ -686,17 +841,30 @@ function install_PowerDB() { echo -e "${GREEN_DARK}To start PowerDB ${NC}: powerd${NC}" fi - if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" - else - echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" - fi + #if [ ${openresty_work} = 'true' ]; then + # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + #else + # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" + #fi if [ ! -z "$firstEp" ]; then - echo - echo -e "${GREEN_DARK}Please run${NC}: power -h $firstEp${GREEN_DARK} to login into cluster, then${NC}" + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $serverFqdn${GREEN_DARK} to login into PowerDB server${NC}" + echo fi echo -e "\033[44;32;1mPowerDB is installed successfully!${NC}" echo @@ -713,6 +881,7 @@ function install_PowerDB() { ## ==============================Main program starts from here============================ +serverFqdn=$(hostname -f) if [ "$verType" == "server" ]; then # Install server and client if [ -x ${bin_dir}/powerd ]; then diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index ee79a560407c650de8c511964f611cf8cacfd2d5..00dfcb75590a50d23180e5eb2b6ad38f28924bba 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index fdb3e0e5cc8c7add9c8ea8cdc9224964b3c80153..509df31297d5b442a05516b37a1b82572ae4f610 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -77,6 +77,7 @@ if [ "$osType" != "Darwin" ]; then cp ${build_dir}/bin/taos ${install_dir}/bin/power cp ${script_dir}/remove_power.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo + cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 5ae5cbbcdc00c1fd862501a6eb0dccd3ac0fa007..0b4659a9112f933a5ba9b91f52f3df203d18a2e7 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 13849484695f98a5fc206d88ec6a5de0cc930a9b..ba57fe5e9687b32b73a0a3ff4d48807f04a0202d 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -77,6 +77,7 @@ else cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd cp ${script_dir}/remove_power.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo + cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${build_dir}/bin/tarbitrator ${install_dir}/bin cp ${script_dir}/set_core.sh ${install_dir}/bin diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 52919976ee8a1b8c3ab7c38d9dad4eb7e98b22bd..9d15b9736ed8647440ee3cd0f25276d635da1952 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -3,6 +3,10 @@ # This file is used to install tdengine rpm package on centos systems. The operating system # is required to use systemd to manage services at boot #set -x + +iplist="" +serverFqdn="" + # -----------------------Variables definition--------------------- script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory @@ -92,6 +96,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -102,16 +107,176 @@ function install_bin() { [ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : [ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : [ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${bin_dir}/taosdemox ] && ${csudo} ln -s ${bin_dir}/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : } +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname -f) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + function install_config() { if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} ${csudo} mkdir -p ${cfg_install_dir} [ -f ${cfg_dir}/taos.cfg ] && ${csudo} cp ${cfg_dir}/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi + + # Save standard input to 6 and open / dev / TTY on standard input + exec 6<&0 0 /dev/null; then csudo="sudo" fi -#ulimit -c unlimited +if [[ ! -n ${corePath} ]]; then + echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:" + read corePath + while true; do + if [[ ! -z "$corePath" ]]; then + break + else + read -p "Please enter a file directory to save the coredump file:" corePath + fi + done +fi + +ulimit -c unlimited ${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||: ${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||: source /etc/profile -${csudo} mkdir -p /coredump ||: -${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p' ||: -${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern ||: +${csudo} mkdir -p ${corePath} ||: +${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: +${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern ||: diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b5d06a4adb2acc57e7220dfde72290fec0eec0fb..d1a334664e17c89dc3f55849f8ceb203e5026fe6 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.9.0' +version: '2.0.12.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.9.0 + - usr/lib/libtaos.so.2.0.12.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index d80488fe9fa35df7af217b26cb93c85f9b11192a..3e1d5eda763c99afb8a0c5dcd3f08996eec80337 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -44,7 +44,7 @@ static void bnUnLock() { static bool bnCheckFree(SDnodeObj *pDnode) { if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { - mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status)); + mError("dnode:%d, status:%s not available", pDnode->dnodeId, dnodeStatus[pDnode->status]); return false; } @@ -66,6 +66,77 @@ static bool bnCheckFree(SDnodeObj *pDnode) { return true; } +static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { + SVnodeGid tmp = *pVnodeGid1; + *pVnodeGid1 = *pVnodeGid2; + *pVnodeGid2 = tmp; +} + +static void bnAdjustVnodeIndex(SVgObj *pInVg) { + int32_t d0Id = pInVg->vnodeGid[0].dnodeId; + int32_t d1Id = pInVg->vnodeGid[1].dnodeId; + int32_t d2Id = pInVg->vnodeGid[2].dnodeId; + + int32_t vgId = pInVg->vgId; + int32_t d0Num = 0; + int32_t d1Num = 0; + int32_t d2Num = 0; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + if (pVgroup->vgId != vgId) { + if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++; + if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++; + if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++; + } + + mnodeDecVgroupRef(pVgroup); + } + + if (pInVg->numOfVnodes == 1) { + } + + if (pInVg->numOfVnodes == 2) { + mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num); + if (d0Num > d1Num) { + mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); + } + } + + if (pInVg->numOfVnodes >= 3) { + mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num); + if (d0Num <= d1Num && d0Num <= d2Num) { + if (d1Num > d2Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } else if (d1Num <= d2Num && d1Num <= d0Num) { + mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); + if (d0Num > d2Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } else { + mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]); + if (d1Num > d0Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } + } + + for (int i = 0; i < pInVg->numOfVnodes; ++i) { + mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId); + } +} + static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId); @@ -88,17 +159,11 @@ static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); pVgroup->numOfVnodes = numOfVnodes; + bnAdjustVnodeIndex(pVgroup); mnodeUpdateVgroup(pVgroup); } -static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { - // SVnodeGid tmp = *pVnodeGid1; - // *pVnodeGid1 = *pVnodeGid2; - // *pVnodeGid2 = tmp; -} - int32_t bnAllocVnodes(SVgObj *pVgroup) { - static int32_t randIndex = 0; int32_t dnode = 0; int32_t vnodes = 0; @@ -120,8 +185,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { break; } else { mDebug("dnode:%d, is not selected, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, - mnodeGetDnodeStatusStr(pDnode->status), pDnode->openVnodes, pDnode->diskAvailable, - pDnode->alternativeRole); + dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); } } } @@ -137,7 +201,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { while (1) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; - mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), + mDebug("dnode:%d, status:%s vnodes:%d disk:%fGB role:%d", pDnode->dnodeId, dnodeStatus[pDnode->status], pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); mnodeDecDnodeRef(pDnode); } @@ -149,36 +213,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { } } - /* - * make the choice more random. - * replica 1: no choice - * replica 2: there are 2 combinations - * replica 3 or larger: there are 6 combinations - */ - if (pVgroup->numOfVnodes == 1) { - } else if (pVgroup->numOfVnodes == 2) { - if (randIndex++ % 2 == 0) { - bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1); - } - } else { - int32_t randVal = randIndex++ % 6; - if (randVal == 1) { // 1, 0, 2 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); - } else if (randVal == 2) { // 1, 2, 0 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); - bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); - } else if (randVal == 3) { // 2, 1, 0 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); - } else if (randVal == 4) { // 2, 0, 1 - bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); - bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); - } - if (randVal == 5) { // 0, 2, 1 - bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); - } else { - } // 0, 1, 2 - } - + bnAdjustVnodeIndex(pVgroup); bnReleaseDnodes(); bnUnLock(); return TSDB_CODE_SUCCESS; @@ -189,17 +224,32 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { return false; } + int32_t rmVnodeVer = 0; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SVnodeGid *pVnode = pVgroup->vnodeGid + i; + if (pVnode == pRmVnode) { + rmVnodeVer = mnodeGetVgidVer(pVnode->vver); + mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i, + pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer); + } + } + bool isReady = false; for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVnode = pVgroup->vnodeGid + i; + SDnodeObj *pDnode = pVnode->pDnode; if (pVnode == pRmVnode) continue; + int32_t vver = mnodeGetVgidVer(pVnode->vver); - mTrace("vgId:%d, check vgroup status, dnode:%d status:%d, vnode role:%s", pVgroup->vgId, pVnode->pDnode->dnodeId, - pVnode->pDnode->status, syncRole[pVnode->role]); - if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue; - if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; + mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d", pVgroup->vgId, i, + pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); + if (pDnode->status == TAOS_DN_STATUS_DROPPING) continue; + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; + if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue; - if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) { + if (rmVnodeVer == 0 || vver >= rmVnodeVer) { + mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d", + pVgroup->vgId, i, pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); isReady = true; } } @@ -214,50 +264,14 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { static int32_t bnRemoveVnode(SVgObj *pVgroup) { if (pVgroup->numOfVnodes <= 1) return -1; - SVnodeGid *pRmVnode = NULL; - SVnodeGid *pSelVnode = NULL; - int32_t maxScore = 0; - - for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SVnodeGid *pVnode = &(pVgroup->vnodeGid[i]); - SDnodeObj *pDnode = mnodeGetDnode(pVnode->dnodeId); - - if (pDnode == NULL) { - mError("vgId:%d, dnode:%d not exist, remove it", pVgroup->vgId, pVnode->dnodeId); - pRmVnode = pVnode; - break; - } - - if (pDnode->status == TAOS_DN_STATUS_DROPPING) { - mDebug("vgId:%d, dnode:%d in dropping state", pVgroup->vgId, pVnode->dnodeId); - pRmVnode = pVnode; - } else if (pVnode->dnodeId == pVgroup->lbDnodeId) { - mDebug("vgId:%d, dnode:%d in updating state", pVgroup->vgId, pVnode->dnodeId); - pRmVnode = pVnode; - } else { - if (pSelVnode == NULL) { - pSelVnode = pVnode; - maxScore = pDnode->score; - } else { - if (maxScore < pDnode->score) { - pSelVnode = pVnode; - maxScore = pDnode->score; - } - } - } - - mnodeDecDnodeRef(pDnode); - } - - if (pRmVnode != NULL) { - pSelVnode = pRmVnode; - } + SVnodeGid *pSelVnode = &pVgroup->vnodeGid[pVgroup->numOfVnodes - 1]; + mDebug("vgId:%d, vnode in dnode:%d will be dropped", pVgroup->vgId, pSelVnode->dnodeId); if (!bnCheckVgroupReady(pVgroup, pSelVnode)) { mDebug("vgId:%d, is not ready", pVgroup->vgId); return -1; } else { - mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); + mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); bnDiscardVnode(pVgroup, pSelVnode); return TSDB_CODE_SUCCESS; } @@ -275,36 +289,48 @@ static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) { return false; } -/** - * desc: add vnode to vgroup, find a new one if dest dnode is null - **/ -static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { - if (pDestDnode == NULL) { - for (int32_t i = 0; i < tsBnDnodes.size; ++i) { - SDnodeObj *pDnode = tsBnDnodes.list[i]; - if (pDnode == pSrcDnode) continue; - if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue; - if (!bnCheckFree(pDnode)) continue; - - pDestDnode = pDnode; - mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId); - break; - } +static SDnodeObj *bnGetAvailDnode(SVgObj *pVgroup) { + for (int32_t i = 0; i < tsBnDnodes.size; ++i) { + SDnodeObj *pDnode = tsBnDnodes.list[i]; + if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue; + if (!bnCheckFree(pDnode)) continue; + + mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId); + return pDnode; } - if (pDestDnode == NULL) { + return NULL; +} + +static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { + if (pDestDnode == NULL || pSrcDnode == pDestDnode) { return TSDB_CODE_MND_DNODE_NOT_EXIST; } - SVnodeGid *pVnodeGid = pVgroup->vnodeGid + pVgroup->numOfVnodes; - pVnodeGid->dnodeId = pDestDnode->dnodeId; - pVnodeGid->pDnode = pDestDnode; - pVgroup->numOfVnodes++; + SVnodeGid vnodeGids[TSDB_MAX_REPLICA]; + memcpy(&vnodeGids, &pVgroup->vnodeGid, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); + + int32_t numOfVnodes = pVgroup->numOfVnodes; + vnodeGids[numOfVnodes].dnodeId = pDestDnode->dnodeId; + vnodeGids[numOfVnodes].pDnode = pDestDnode; + numOfVnodes++; + + // move the src vnode to the end + for (int32_t v = 0; v < numOfVnodes; ++v) { + if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) { + bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]); + pVgroup->lbDnodeId = pSrcDnode->dnodeId; + break; + } + } - if (pSrcDnode != NULL) { - pVgroup->lbDnodeId = pSrcDnode->dnodeId; + // adjust the vgroup postion + if (pSrcDnode == NULL) { + bnAdjustVnodeIndex(pVgroup); } + memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); + pVgroup->numOfVnodes = numOfVnodes; atomic_add_fetch_32(&pDestDnode->openVnodes, 1); mnodeUpdateVgroup(pVgroup); @@ -315,16 +341,16 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes static bool bnMonitorBalance() { if (tsBnDnodes.size < 2) return false; + mDebug("monitor dnodes for balance, avail:%d", tsBnDnodes.size); for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) { SDnodeObj *pDnode = tsBnDnodes.list[src]; - mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1, - pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores, - pDnode->openVnodes); + mDebug("%d-dnode:%d, state:%s, score:%.1f, cores:%d, vnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId, + dnodeStatus[pDnode->status], pDnode->score, pDnode->numOfCores, pDnode->openVnodes); } float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score; if (scoresDiff < 0.01) { - mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff); + mDebug("all dnodes:%d is already balanced, scoreDiff:%.1f", tsBnDnodes.size, scoresDiff); return false; } @@ -392,7 +418,7 @@ void bnReset() { tsAccessSquence = 0; } -static int32_t bnMonitorVgroups() { +static bool bnMonitorVgroups() { void * pIter = NULL; SVgObj *pVgroup = NULL; bool hasUpdatingVgroup = false; @@ -412,7 +438,13 @@ static int32_t bnMonitorVgroups() { } else if (vgReplica < dbReplica) { mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica); hasUpdatingVgroup = true; - code = bnAddVnode(pVgroup, NULL, NULL); + + SDnodeObj *pAvailDnode = bnGetAvailDnode(pVgroup); + if (pAvailDnode == NULL) { + code = TSDB_CODE_MND_DNODE_NOT_EXIST; + } else { + code = bnAddVnode(pVgroup, NULL, pAvailDnode); + } } mnodeDecVgroupRef(pVgroup); @@ -545,6 +577,7 @@ void bnCheckStatus() { mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence, pDnode->lastAccess, pDnode->status); bnSetVgroupOffline(pDnode); + bnStartTimer(3000); } } mnodeDecDnodeRef(pDnode); diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c index e5ad7a211958dce9507c5fca22f57a6d99c82029..cbc2c6218458a1ec194ff49f215b80a5195d2271 100644 --- a/src/balance/src/bnScore.c +++ b/src/balance/src/bnScore.c @@ -299,7 +299,7 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status)); + STR_TO_VARSTR(pWrite, dnodeStatus[pDnode->status]); cols++; numOfRows++; diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index bf046a9faea6e9e3d7b324e7786c594d16a9aeb7..84f8694fca7248abb27529f5e8268dc0e08bf815 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -31,7 +31,10 @@ static void *bnThreadFunc(void *arg) { } pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); + mDebug("balance thread wakes up to work"); bool updateSoon = bnStart(); + mDebug("balance thread finished this poll, updateSoon:%d", updateSoon); + bnStartTimer(updateSoon ? 1000 : -1); pthread_mutex_unlock(&(tsBnThread.mutex)); } @@ -101,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) { tsBnThread.timer = NULL; tsAccessSquence++; - bnCheckStatus(); bnStartTimer(-1); + bnCheckStatus(); if (handle == NULL) { if (tsAccessSquence % tsBalanceInterval == 0) { @@ -121,6 +124,7 @@ void bnStartTimer(int64_t mseconds) { bool updateSoon = (mseconds != -1); if (updateSoon) { + mTrace("balance function will be called after %" PRId64 " ms", mseconds); taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 43ba31f331fc5c92d4142a008db458b23395a473..06176451881b30ebe4c24bfcd478ceeb221a86de 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -38,12 +38,6 @@ typedef struct SLocalDataSource { tFilePage filePage; } SLocalDataSource; -enum { - TSC_LOCALREDUCE_READY = 0x0, - TSC_LOCALREDUCE_IN_PROGRESS = 0x1, - TSC_LOCALREDUCE_TOBE_FREED = 0x2, -}; - typedef struct SLocalReducer { SLocalDataSource ** pLocalDataSrc; int32_t numOfBuffer; @@ -56,7 +50,6 @@ typedef struct SLocalReducer { tFilePage * pTempBuffer; struct SQLFunctionCtx *pCtx; int32_t rowSize; // size of each intermediate result. - int32_t status; // denote it is in reduce process, in reduce process, it bool hasPrevRow; // cannot be released bool hasUnprocessedRow; tOrderDescriptor * pDesc; diff --git a/src/client/inc/tscLog.h b/src/client/inc/tscLog.h index 9d01edae3680c2b6aa3c075229ef077eebbe230b..5273a87ea0d0549420acf2b6679f50ce22159ebc 100644 --- a/src/client/inc/tscLog.h +++ b/src/client/inc/tscLog.h @@ -23,15 +23,15 @@ extern "C" { #include "tlog.h" extern int32_t cDebugFlag; -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; -#define tscFatal(...) { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscError(...) { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscWarn(...) { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscInfo(...) { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} -#define tscDebug(...) { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} -#define tscTrace(...) { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} -#define tscDebugL(...){ if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} +#define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) +#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index eddfa62966113d1a2befd2ed409d04c1e0665a61..8d82c65cee5cb26c3a4a37e6bbe6076dae28aaa5 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -20,9 +20,6 @@ extern "C" { #endif -/* - * @date 2018/09/30 - */ #include "exception.h" #include "os.h" #include "qExtbuffer.h" @@ -216,7 +213,7 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); @@ -234,7 +231,7 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); -int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); +int tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); void tscResetForNextRetrieve(SSqlRes* pRes); void tscDoQuery(SSqlObj* pSql); @@ -276,7 +273,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreClauseToTry(SSqlObj* pSql); -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache); +void tscFreeQueryInfo(SSqlCmd* pCmd); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -287,6 +284,17 @@ bool tscSetSqlOwner(SSqlObj* pSql); void tscClearSqlOwner(SSqlObj* pSql); int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize); +char* serializeTagData(STagData* pTagData, char* pMsg); +int32_t copyTagData(STagData* dst, const STagData* src); + +STableMeta* createSuperTableMeta(STableMetaMsg* pChild); +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); +uint32_t tscGetTableMetaMaxSize(); +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name); +STableMeta* tscTableMetaClone(STableMeta* pTableMeta); + + void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); char* strdup_throw(const char* str); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index f6dc45398f35c38598f3f3132b2f6f5601a4ed68..7c41164a046165c785d3d4e0e0a8979854e4b6f2 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -105,7 +105,10 @@ SSchema tscGetTbnameColumnSchema(); * @param size size of the table meta * @return */ -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg); + +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bdb35cb07274a880c98aa2c93005c47fe05c7de2..25a299d0988144980dded10b170cbe7b7c662b04 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -56,22 +56,28 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; -typedef struct SCorVgroupInfo { - int32_t version; - int8_t inUse; - int8_t numOfEps; - SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCorVgroupInfo; +typedef struct SNewVgroupInfo { + int32_t vgId; + int8_t inUse; + int8_t numOfEps; + SEpAddrMsg ep[TSDB_MAX_REPLICA]; +} SNewVgroupInfo; + +typedef struct CChildTableMeta { + int32_t vgId; + STableId id; + uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; +} CChildTableMeta; typedef struct STableMeta { - STableComInfo tableInfo; + int32_t vgId; + STableId id; uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; int16_t sversion; int16_t tversion; - char sTableId[TSDB_TABLE_FNAME_LEN]; - SVgroupInfo vgroupInfo; - SCorVgroupInfo corVgroupInfo; - STableId id; + STableComInfo tableInfo; SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; @@ -170,7 +176,7 @@ typedef struct SParamInfo { } SParamInfo; typedef struct STableDataBlocks { - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableName[TSDB_TABLE_FNAME_LEN]; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id @@ -229,7 +235,7 @@ typedef struct { int32_t numOfTablesInSubmit; }; - int32_t insertType; + uint32_t insertType; int32_t clauseIndex; // index of multiple subclause query char * curSql; // current sql, resume position of sql after parsing paused @@ -246,9 +252,9 @@ typedef struct { int8_t dataSourceType; // load data from file or not int8_t submitSchema; // submit block is built with table schema - STagData *pTagData; // NOTE: pTagData->data is used as a variant length array + STagData tagData; // NOTE: pTagData->data is used as a variant length array - STableMeta **pTableMetaList; // all involved tableMeta list of current insert sql statement. + char **pTableNameList; // all involved tableMeta list of current insert sql statement. int32_t numOfTables; SHashObj *pTableBlockHashList; // data block for each table @@ -285,8 +291,8 @@ typedef struct { char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex* pColumnIndex; - SArithmeticSupport* pArithSup; // support the arithmetic expression calculation on agg functions - struct SLocalReducer* pLocalReducer; + SArithmeticSupport *pArithSup; // support the arithmetic expression calculation on agg functions + struct SLocalReducer *pLocalReducer; } SSqlRes; typedef struct STscObj { @@ -307,7 +313,7 @@ typedef struct STscObj { SRpcCorEpSet *tscCorMgmtEpSet; void* pDnodeConn; pthread_mutex_t mutex; - T_REF_DECLARE() + int32_t numOfObj; // number of sqlObj from this tscObj } STscObj; typedef struct SSubqueryState { @@ -385,7 +391,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); int tscProcessSql(SSqlObj *pSql); int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex); -void tscQueueAsyncRes(SSqlObj *pSql); +void tscAsyncResultOnError(SSqlObj *pSql); void tscQueueAsyncError(void(*fp), void *param, int32_t code); @@ -399,7 +405,7 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); +void tscResetSqlCmdObj(SSqlCmd *pCmd); /** * free query result of the sql object @@ -413,14 +419,13 @@ void tscFreeSqlResult(SSqlObj *pSql); */ void tscFreeSqlObj(SSqlObj *pSql); void tscFreeRegisteredSqlObj(void *pSql); -void tscFreeTableMetaHelper(void *pTableMeta); void tscCloseTscObj(void *pObj); // todo move to taos? or create a new file: taos_internal.h TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos); -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res); +TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res); void waitForQueryRsp(void *param, TAOS_RES *tres, int code); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen); @@ -478,15 +483,16 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } } -extern SCacheObj* tscMetaCache; -extern int tscObjRef; -extern void * tscTmr; -extern void * tscQhandle; -extern int tscKeepConn[]; -extern int tsInsertHeadSize; -extern int tscNumOfThreads; -extern int tscRefId; - +extern int32_t sentinel; +extern SHashObj *tscVgroupMap; +extern SHashObj *tscTableMetaInfo; + +extern int tscObjRef; +extern void *tscTmr; +extern void *tscQhandle; +extern int tscKeepConn[]; +extern int tscRefId; +extern int tscNumOfObj; // number of existed sqlObj in current process. extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 910a7b41126536eb0e7838e8aaf196b64642e5f4..96aeb9d60de1ab6fbaeebcb54e2da1ab316179f8 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -18,7 +18,6 @@ #include "tnote.h" #include "trpc.h" -#include "tcache.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscLocalMerge.h" @@ -57,7 +56,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -71,7 +70,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -166,7 +165,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pRes->code = numOfRows; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -217,7 +216,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; pSql->param = param; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -280,7 +279,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), pSql->param = param; pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -381,16 +380,20 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) { taosScheduleTask(tscQhandle, &schedMsg); } -void tscQueueAsyncRes(SSqlObj *pSql) { + +void tscAsyncResultOnError(SSqlObj *pSql) { if (pSql == NULL || pSql->signature != pSql) { tscDebug("%p SqlObj is freed, not add into queue async res", pSql); return; } + assert(pSql->res.code != TSDB_CODE_SUCCESS); tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code)); SSqlRes *pRes = &pSql->res; - assert(pSql->fp != NULL && pSql->fetchFp != NULL); + if (pSql->fp == NULL || pSql->fetchFp == NULL){ + return; + } pSql->fp = pSql->fetchFp; (*pSql->fp)(pSql->param, pSql, pRes->code); @@ -419,7 +422,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { // check if it is a sub-query of super table query first, if true, enter another routine if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) { - tscDebug("%p update table meta in local cache, continue to process sql and send the corresponding query", pSql); + tscDebug("%p update local table meta, continue to process sql and send the corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -436,7 +439,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { return; } else { // continue to process normal async query if (pCmd->parseFinished) { - tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql); + tscDebug("%p update local table meta, continue to process sql and send corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -451,7 +454,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pCmd->command == TSDB_SQL_SELECT) { tscDebug("%p redo parse sql string and proceed", pSql); pCmd->parseFinished = false; - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmdObj(pCmd); code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -528,6 +531,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { _error: if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 4c28adc261f6b83b3c1bbb68d3dd3d56e6007fc9..16bbd420c0cd77bd3f94ce64d840ccda36a36eb6 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -17,7 +17,6 @@ #include "taosmsg.h" #include "taosdef.h" -#include "tcache.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" @@ -46,7 +45,8 @@ typedef struct SCreateBuilder { SSqlObj *pInterSql; int32_t (*fp)(void *para, char* result); Stage callStage; -} SCreateBuilder; +} SCreateBuilder; + static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { @@ -207,10 +207,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { const int32_t TYPE_COLUMN_LENGTH = 16; const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql); -// if (noteFieldLen == 0) { -// noteFieldLen = NOTE_COLUMN_MIN_LENGTH; -// } + int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH; int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); tscFieldInfoUpdateOffset(pQueryInfo); @@ -275,7 +272,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code != TSDB_CODE_SUCCESS) { taos_free_result(pSql); free(builder); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -293,7 +290,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, code); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -571,9 +568,9 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch return TSDB_CODE_TSC_OUT_OF_MEMORY; } - char fullName[TSDB_TABLE_FNAME_LEN] = {0}; + char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0}; extractDBName(pTableMetaInfo->name, fullName); - extractTableName(pMeta->sTableId, param->sTableName); + extractTableName(pMeta->sTableName, param->sTableName); snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName); extractTableName(pTableMetaInfo->name, param->buf); @@ -822,26 +819,39 @@ static int32_t tscProcessClientVer(SSqlObj *pSql) { } +// TODO add test cases. +static int32_t checkForOnlineNode(SSqlObj* pSql) { + int32_t* data = pSql->res.length; + if (data == NULL) { + return TSDB_CODE_SUCCESS; + } + + int32_t total = data[0]; + int32_t online = data[1]; + return (online < total)? TSDB_CODE_RPC_NETWORK_UNAVAIL:TSDB_CODE_SUCCESS; +} + static int32_t tscProcessServStatus(SSqlObj *pSql) { STscObj* pObj = pSql->pTscObj; SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); if (pHb != NULL) { - int32_t code = pHb->res.code; + pSql->res.code = pHb->res.code; taosReleaseRef(tscObjRef, pObj->hbrid); - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - return pSql->res.code; - } - } else { - if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - return pSql->res.code; - } } - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return pSql->res.code; + } + pSql->res.code = checkForOnlineNode(pHb); + if (pSql->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return pSql->res.code; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); + int32_t val = 1; tscSetLocalQueryResult(pSql, (char*) &val, pExpr->aliasName, TSDB_DATA_TYPE_INT, sizeof(int32_t)); return TSDB_CODE_SUCCESS; @@ -890,7 +900,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { pRes->code = tscProcessShowCreateDatabase(pSql); } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosCacheEmpty(tscMetaCache); + taosHashEmpty(tscTableMetaInfo); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); @@ -914,7 +924,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { (*pSql->fp)(pSql->param, pSql, code); } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){ } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index a99918975e11a1f51bc93eebf6053cfd8dca05c3..921aa9bade64759d149769726db0a5381c4b112c 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -89,11 +89,11 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->startOffset = 0; pCtx->size = 1; pCtx->hasNull = true; - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; // for top/bottom function, the output of timestamp is the first column int32_t functionId = pExpr->functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; @@ -493,13 +493,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { // there is no more result, so we release all allocated resource SLocalReducer *pLocalReducer = (SLocalReducer *)atomic_exchange_ptr(&pRes->pLocalReducer, NULL); if (pLocalReducer != NULL) { - int32_t status = 0; - while ((status = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, - TSC_LOCALREDUCE_TOBE_FREED)) == TSC_LOCALREDUCE_IN_PROGRESS) { - taosMsleep(100); - tscDebug("%p waiting for delete procedure, status: %d", pSql, status); - } - pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo); if (pLocalReducer->pCtx != NULL) { @@ -911,6 +904,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, } } + if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) { + pRes->numOfRows = 0; + pBeforeFillData->num = 0; + pLocalReducer->discard = true; + return; + } + pRes->numOfRowsGroup += pRes->numOfRows; // impose the limitation of output rows on the final result @@ -1067,7 +1067,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, pCtx->param[0].i64Key = pExpr->param[0].i64Key; } - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; if (needInit) { aAggs[pCtx->functionId].init(pCtx); @@ -1080,7 +1080,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, continue; } - aAggs[functionId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); + aAggs[functionId].mergeFunc(&pLocalReducer->pCtx[j]); } } @@ -1296,6 +1296,10 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// re for (int32_t i = 0; i < t; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity; + + if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM || pExpr->functionId == TSDB_FUNC_DIFF) { + pLocalReducer->pCtx[i].ptsOutputBuf = pLocalReducer->pCtx[0].aOutputBuf; + } } memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage)); @@ -1430,24 +1434,13 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { SLocalReducer *pLocalReducer = pRes->pLocalReducer; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - // set the data merge in progress - int32_t prevStatus = - atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); - if (prevStatus != TSC_LOCALREDUCE_READY) { - assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already - return TSDB_CODE_SUCCESS; - } - - tFilePage *tmpBuffer = pLocalReducer->pTempBuffer; + tFilePage *tmpBuffer = pLocalReducer->pTempBuffer; if (doHandleLastRemainData(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } if (doBuildFilledResultForGroup(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } @@ -1503,7 +1496,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { pLocalReducer->discardData->num = 0; if (saveGroupResultInfo(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; return TSDB_CODE_SUCCESS; } @@ -1549,7 +1541,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { // here we do not check the return value adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); - assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS); if (pRes->numOfRows == 0) { handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); @@ -1560,7 +1551,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * If previous group is not skipped, keep it in pRes->numOfGroups */ if (notSkipped && saveGroupResultInfo(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; return TSDB_CODE_SUCCESS; } @@ -1580,7 +1570,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { if (pRes->numOfRows == 0) { continue; } else { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } } else { // result buffer is not full @@ -1605,9 +1594,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { genFinalResults(pSql, pLocalReducer, true); } - assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS && pRes->row == 0); - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. - return TSDB_CODE_SUCCESS; } @@ -1661,7 +1647,7 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ // calculate the result from several other columns if (pSup->pArithExprInfo != NULL) { arithSup.pArithExpr = pSup->pArithExprInfo; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); } else { SSqlExpr* pExpr = pSup->pSqlExpr; memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num)); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 9d04a5c13a5befd02517f94eedf7724a85fa8326..ec90d21394a5dce657652fb9e6c2ca81dd2f120f 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -731,7 +731,7 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColI return code; } - dataBuf->vgId = pTableMeta->vgroupInfo.vgId; + dataBuf->vgId = pTableMeta->vgId; dataBuf->numOfTables = 1; *totalNum += numOfRows; @@ -796,8 +796,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sToken = tStrGetToken(sql, &index, false, 0, NULL); sql += index; - tscAllocPayload(pCmd, sizeof(STagData)); - //the source super table is moved to the secondary position of the pTableMetaInfo list if (pQueryInfo->numOfTables < 2) { tscAddEmptyMetaInfo(pQueryInfo); @@ -809,13 +807,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } - STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data)); - if (pTag == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memset(pTag, 0, offsetof(STagData, data)); - tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name)); - pCmd->pTagData = pTag; + tstrncpy(pCmd->tagData.name, pSTableMeterMetaInfo->name, sizeof(pCmd->tagData.name)); + pCmd->tagData.dataLen = 0; code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); if (code != TSDB_CODE_SUCCESS) { @@ -946,14 +939,15 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } tdSortKVRowByColIdx(row); - pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row)); + pCmd->tagData.dataLen = kvRowLen(row); + char* pTag = realloc(pCmd->tagData.data, pCmd->tagData.dataLen); if (pTag == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pCmd->pTagData = pTag; - pTag->dataLen = htonl(kvRowLen(row)); - kvRowCpy(pTag->data, row); + + kvRowCpy(pTag, row); free(row); + pCmd->tagData.data = pTag; index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); @@ -972,7 +966,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } createTable = true; - code = tscGetMeterMetaEx(pSql, pTableMetaInfo, true); + code = tscGetTableMetaEx(pSql, pTableMetaInfo, true); if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { return code; } @@ -983,7 +977,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } else { sql = sToken.z; } - code = tscGetMeterMetaEx(pSql, pTableMetaInfo, false); + code = tscGetTableMetaEx(pSql, pTableMetaInfo, false); if (pCmd->curSql == NULL) { assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS); @@ -1298,7 +1292,6 @@ int tsInsertInitialCheck(SSqlObj *pSql) { pCmd->count = 0; pCmd->command = TSDB_SQL_INSERT; - pSql->res.numOfRows = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); @@ -1346,7 +1339,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { free(sqlstr); } else { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmdObj(pCmd); free(pSql->sqlstr); pSql->sqlstr = sqlstr; pSql->parseRetry++; @@ -1358,12 +1351,12 @@ int tsParseSql(SSqlObj *pSql, bool initial) { SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmdObj(pCmd); pSql->parseRetry++; ret = tscToSQLCmd(pSql, &SQLInfo); } - SQLInfoDestroy(&SQLInfo); + SqlInfoDestroy(&SQLInfo); } /* @@ -1420,13 +1413,25 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // handle error assert(taos_errno(pSql) == code); - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); + do { + if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + assert(pSql->res.numOfRows == 0); + int32_t errc = fseek(fp, 0, SEEK_SET); + if (errc < 0) { + tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno)); + } else { + break; + } + } - pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); - return; + taos_free_result(pSql); + tfree(pSupporter); + fclose(fp); + + pParentSql->res.code = code; + tscAsyncResultOnError(pParentSql); + return; + } while (0); } // accumulate the total submit records @@ -1446,18 +1451,21 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { int32_t count = 0; int32_t maxRows = 0; - tscDestroyBlockArrayList(pSql->cmd.pDataBlocks); - pCmd->pDataBlocks = taosArrayInit(1, POINTER_BYTES); + tfree(pCmd->pTableNameList); + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + } STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, tinfo.rowSize, sizeof(SSubmitBlk), pTableMetaInfo->name, pTableMeta, &pTableDataBlock); + int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, + sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL); if (ret != TSDB_CODE_SUCCESS) { // return ret; } - taosArrayPush(pCmd->pDataBlocks, &pTableDataBlock); tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows); - char *tokenBuf = calloc(1, 4096); while ((readLen = tgetline(&line, &n, fp)) != -1) { @@ -1492,7 +1500,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { code = doPackSendDataBlock(pSql, count, pTableDataBlock); if (code != TSDB_CODE_SUCCESS) { pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1519,8 +1527,6 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { SImportFileSupport *pSupporter = calloc(1, sizeof(SImportFileSupport)); SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); - - pNew->cmd.pDataBlocks = taosArrayInit(4, POINTER_BYTES); pCmd->count = 1; FILE *fp = fopen(pCmd->payload, "r"); @@ -1528,8 +1534,8 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); - tfree(pSupporter) - tscQueueAsyncRes(pSql); + tfree(pSupporter); + tscAsyncResultOnError(pSql); return; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c74d0ef2cf14ed2fcd7520a870ca48c51f059039..4a5a7b09a2013fe76cd4abcbc2f0fb903758d935 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -66,9 +66,9 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); -static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); +static void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength); -static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult); +static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); @@ -87,7 +87,7 @@ static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuery static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem); +static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem); static int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql); static int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL); @@ -666,6 +666,7 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; const char* msg3 = "sliding cannot be used without interval"; + const char* msg4 = "top/bottom query does not support order by value in interval query"; SSqlCmd* pCmd = &pSql->cmd; @@ -712,6 +713,11 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } + int32_t colId = pQueryInfo->order.orderColId; + if (pQueryInfo->interval.interval > 0 && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + return TSDB_CODE_SUCCESS; } @@ -904,7 +910,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa * that are corresponding to the old name for the new table name. */ if (strlen(oldName) > 0 && strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearTableMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo); } return TSDB_CODE_SUCCESS; @@ -996,33 +1002,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC return false; } - int32_t nLen = 0; - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - if (p->bytes == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - nLen += p->bytes; - } - - // max tag row length must be less than TSDB_MAX_TAGS_LEN - if (nLen > TSDB_MAX_TAGS_LEN) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - // field name must be unique - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - - if (has(pFieldList, 0, p->name) == true) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - } - /* timestamp in tag is not allowed */ for (int32_t i = 0; i < numOfTags; ++i) { TAOS_FIELD* p = taosArrayGet(pTagsList, i); @@ -1054,6 +1033,33 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC } } + int32_t nLen = 0; + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + if (p->bytes == 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return false; + } + + nLen += p->bytes; + } + + // max tag row length must be less than TSDB_MAX_TAGS_LEN + if (nLen > TSDB_MAX_TAGS_LEN) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return false; + } + + // field name must be unique + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + + if (has(pFieldList, 0, p->name) == true) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return false; + } + } + return true; } @@ -1279,7 +1285,7 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* tscColumnListInsert(pQueryInfo->colList, &tsCol); } -static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { +static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSqlExprItem* pItem) { const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; const char* msg2 = "invalid arithmetic expression in select clause"; const char* msg3 = "tag columns can not be used in arithmetic expression"; @@ -1420,7 +1426,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t return TSDB_CODE_SUCCESS; } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSqlExprItem* pItem) { SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, pIndex->columnIndex, pIndex->tableIndex); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); @@ -1484,7 +1490,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel for (int32_t i = 0; i < pSelection->nExpr; ++i) { int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tSQLExprItem* pItem = &pSelection->a[i]; + tSqlExprItem* pItem = &pSelection->a[i]; // project on all fields int32_t optr = pItem->pNode->nSQLOptr; @@ -1643,7 +1649,7 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { +int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; @@ -1767,7 +1773,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return TSDB_CODE_SUCCESS; } -void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { +void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { if (pItem->aliasName != NULL) { tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); } else if (multiCols) { @@ -1775,19 +1781,22 @@ void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrT int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); - int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1; - char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0}; - - snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); + if (tsKeepOriginalColumnName) { // keep the original column name + tstrncpy(name, uname, TSDB_COL_NAME_LEN); + } else { + int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1; + char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0}; + snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); - tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + } } else { // use the user-input result column name int32_t len = MIN(pItem->pNode->token.n + 1, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->pNode->token.z, len); } } -int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { +int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; @@ -1817,7 +1826,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - tSQLExprItem* pParamElem = &pItem->pNode->pParam->a[0]; + tSqlExprItem* pParamElem = &pItem->pNode->pParam->a[0]; SStrToken* pToken = &pParamElem->pNode->colInfo; int16_t sqlOptr = pParamElem->pNode->nSQLOptr; if ((pToken->z == NULL || pToken->n == 0) @@ -1918,7 +1927,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); + tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2037,7 +2046,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col /* in first/last function, multiple columns can be add to resultset */ for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) { - tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); + tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2150,7 +2159,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tSQLExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); + tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); if (pParamElem->pNode->nSQLOptr != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2350,7 +2359,7 @@ static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t column return columnList; } -void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength) { +void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLength) { if (pItem->aliasName != NULL) { strncpy(resultFieldName, pItem->aliasName, nameLength); } else { @@ -3517,7 +3526,7 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL}; + tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL}; // sql function list in selection clause. // Append the sqlExpr into exprList of pQueryInfo structure sequentially @@ -3734,7 +3743,7 @@ static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg return invalidSqlErrMsg(msgBuf, msg); } - *parent = tSQLExprCreate((*parent), pExpr, parentOptr); + *parent = tSqlExprCreate((*parent), pExpr, parentOptr); } else { *parent = pExpr; } @@ -3782,7 +3791,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL * to release expression, e.g., m1.ts = m2.ts, * since this expression is used to set the join query type */ - tSQLExprDestroy(*pExpr); + tSqlExprDestroy(*pExpr); } else { ret = setExprToCond(&pCondExpr->pTimewindow, *pExpr, msg3, parentOptr, pQueryInfo->msg); } @@ -3928,17 +3937,17 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) { if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && ((*pExpr)->nSQLOptr == TK_OR || (*pExpr)->nSQLOptr == TK_AND)) { - tSQLExprNodeDestroy(*pExpr); + tSqlExprNodeDestroy(*pExpr); *pExpr = NULL; } else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) { tSQLExpr* tmpPtr = (*pExpr)->pRight; - tSQLExprNodeDestroy(*pExpr); + tSqlExprNodeDestroy(*pExpr); (*pExpr) = tmpPtr; } else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) { tSQLExpr* tmpPtr = (*pExpr)->pLeft; - tSQLExprNodeDestroy(*pExpr); + tSqlExprNodeDestroy(*pExpr); (*pExpr) = tmpPtr; } @@ -3961,7 +3970,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* (*pExpr) = NULL; } else { - *pOut = tSQLExprCreate(NULL, NULL, (*pExpr)->nSQLOptr); + *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->nSQLOptr); doExtractExprForSTable(pCmd, &(*pExpr)->pLeft, pQueryInfo, &((*pOut)->pLeft), tableIndex); doExtractExprForSTable(pCmd, &(*pExpr)->pRight, pQueryInfo, &((*pOut)->pRight), tableIndex); @@ -4010,6 +4019,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pExpr->nSQLOptr == TK_LIKE) { char* str = taosStringBuilderGetResult(sb, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); return TSDB_CODE_SUCCESS; } @@ -4059,6 +4069,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* str = taosStringBuilderGetResult(&sb1, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); taosStringBuilderDestroy(&sb1); tfree(segments); @@ -4168,23 +4179,23 @@ static int32_t validateJoinExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr static void cleanQueryExpr(SCondExpr* pCondExpr) { if (pCondExpr->pTableCond) { - tSQLExprDestroy(pCondExpr->pTableCond); + tSqlExprDestroy(pCondExpr->pTableCond); } if (pCondExpr->pTagCond) { - tSQLExprDestroy(pCondExpr->pTagCond); + tSqlExprDestroy(pCondExpr->pTagCond); } if (pCondExpr->pColumnCond) { - tSQLExprDestroy(pCondExpr->pColumnCond); + tSqlExprDestroy(pCondExpr->pColumnCond); } if (pCondExpr->pTimewindow) { - tSQLExprDestroy(pCondExpr->pTimewindow); + tSqlExprDestroy(pCondExpr->pTimewindow); } if (pCondExpr->pJoinExpr) { - tSQLExprDestroy(pCondExpr->pJoinExpr); + tSqlExprDestroy(pCondExpr->pJoinExpr); } } @@ -4252,8 +4263,8 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); doCompactQueryExpr(pExpr); - - tSQLExprDestroy(p1); + + tSqlExprDestroy(p1); tExprTreeDestroy(&p, NULL); taosArrayDestroy(colList); @@ -4469,6 +4480,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery const char* msg = "illegal value or data overflow"; const char* msg1 = "value is expected"; const char* msg2 = "invalid fill option"; + const char* msg3 = "top/bottom not support fill"; if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); @@ -4551,6 +4563,14 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + return TSDB_CODE_SUCCESS; } @@ -4643,7 +4663,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu if (!(orderByTags || orderByTS) && !isTopBottomQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else { + } else { // order by top/bottom result value column is not supported in case of interval query. assert(!(orderByTags && orderByTS)); } @@ -4910,6 +4930,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pItem = taosArrayGet(pVarList, 1); SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex); + pAlterSQL->tagData.data = calloc(1, pTagsSchema->bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg13); } @@ -4931,7 +4953,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId); + pUpdateMsg->head.vgId = htonl(pTableMeta->vgId); pUpdateMsg->tid = htonl(pTableMeta->id.tid); pUpdateMsg->uid = htobe64(pTableMeta->id.uid); pUpdateMsg->colId = htons(pTagsSchema->colId); @@ -5437,6 +5459,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->quorum = pCreateDb->quorum; pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->update = pCreateDb->update; + pMsg->cacheLastRow = pCreateDb->cachelast; } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { @@ -5913,25 +5936,33 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ if (pExprList->nExpr != 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - + bool server_status = false; tSQLExpr* pExpr = pExprList->a[0].pNode; if (pExpr->operand.z == NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - + //handle 'select 1' + if (pExpr->token.n == 1 && 0 == strncasecmp(pExpr->token.z, "1", 1)) { + server_status = true; + } else { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + } // TODO redefine the function - SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, - {"server_version()", 16}, - {"server_status()", 15}, - {"client_version()", 16}, - {"current_user()", 14}}; + SDNodeDynConfOption functionsInfo[5] = {{"database()", 10}, + {"server_version()", 16}, + {"server_status()", 15}, + {"client_version()", 16}, + {"current_user()", 14}}; int32_t index = -1; - for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { - if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && - functionsInfo[i].len == pExpr->operand.n) { - index = i; - break; + if (server_status == true) { + index = 2; + } else { + for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { + if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && + functionsInfo[i].len == pExpr->operand.n) { + index = i; + break; + } } } @@ -6141,96 +6172,105 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { const int32_t TABLE_INDEX = 0; const int32_t STABLE_INDEX = 1; - STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); + STableMetaInfo* pStableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst - SStrToken* pToken = &(pCreateTable->usingInfo.stableName); + int32_t numOfTables = (int32_t) taosArrayGetSize(pCreateTable->childTableInfo); + for(int32_t j = 0; j < numOfTables; ++j) { + SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j); - if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + SStrToken* pToken = &pCreateTableInfo->stableName; + if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } - int32_t code = tscSetTableFullName(pStableMeterMetaInfo, pToken, pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t code = tscSetTableFullName(pStableMetaInfo, pToken, pSql); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - // get meter meta from mnode - tstrncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, sizeof(pCreateTable->usingInfo.tagdata.name)); - SArray* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals; + // get table meta from mnode + tstrncpy(pCreateTableInfo->tagdata.name, pStableMetaInfo->name, tListLen(pCreateTableInfo->tagdata.name)); + SArray* pList = pCreateTableInfo->pTagVals; - code = tscGetTableMeta(pSql, pStableMeterMetaInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tscGetTableMeta(pSql, pStableMetaInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - size_t size = taosArrayGetSize(pList); - if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != size) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); - } + size_t size = taosArrayGetSize(pList); + if (tscGetNumOfTags(pStableMetaInfo->pTableMeta) != size) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } - // too long tag values will return invalid sql, not be truncated automatically - SSchema* pTagSchema = tscGetTableTagSchema(pStableMeterMetaInfo->pTableMeta); + // too long tag values will return invalid sql, not be truncated automatically + SSchema *pTagSchema = tscGetTableTagSchema(pStableMetaInfo->pTableMeta); + STagData *pTag = &pCreateTableInfo->tagdata; - STagData* pTag = &pCreateTable->usingInfo.tagdata; - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } - int32_t ret = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < size; ++i) { - SSchema* pSchema = &pTagSchema[i]; - tVariantListItem* pItem = taosArrayGet(pList, i); + int32_t ret = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < size; ++i) { + SSchema* pSchema = &pTagSchema[i]; + tVariantListItem* pItem = taosArrayGet(pList, i); - char tagVal[TSDB_MAX_TAGS_LEN]; - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->pVar.nLen > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + char tagVal[TSDB_MAX_TAGS_LEN]; + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + if (pItem->pVar.nLen > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } } - } - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - // check again after the convert since it may be converted from binary to nchar. - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - int16_t len = varDataTLen(tagVal); - if (len > pSchema->bytes) { + // check again after the convert since it may be converted from binary to nchar. + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + int16_t len = varDataTLen(tagVal); + if (len > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - } - if (ret != TSDB_CODE_SUCCESS) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); + tdDestroyKVRowBuilder(&kvRowBuilder); + if (row == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tdSortKVRowByColIdx(row); + pTag->dataLen = kvRowLen(row); + if (pTag->data == NULL) { + pTag->data = malloc(pTag->dataLen); + } - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); - } + kvRowCpy(pTag->data, row); + free(row); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); - if (row == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - tdSortKVRowByColIdx(row); - pTag->dataLen = kvRowLen(row); - kvRowCpy(pTag->data, row); - free(row); + // table name + if (tscValidateName(&(pCreateTableInfo->name)) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } - // table name - if (tscValidateName(&pInfo->pCreateTableInfo->name) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); + ret = tscSetTableFullName(pTableMetaInfo, &pCreateTableInfo->name, pSql); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } - STableMetaInfo* pTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX); - ret = tscSetTableFullName(pTableMeterMetaInfo, &pInfo->pCreateTableInfo->name, pSql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + pCreateTableInfo->fullname = strndup(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN); } return TSDB_CODE_SUCCESS; @@ -6343,18 +6383,52 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_SUCCESS; } +static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { + const char* msg3 = "start(end) time of query range required or time range too large"; + + if (pQueryInfo->interval.interval == 0) { + return TSDB_CODE_SUCCESS; + } + + bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); + if (initialWindows) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + + int64_t intervalRange = 0; + if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { + int64_t f = 1; + if (pQueryInfo->interval.intervalUnit == 'n') { + f = 30L * MILLISECOND_PER_DAY; + } else if (pQueryInfo->interval.intervalUnit == 'y') { + f = 365L * MILLISECOND_PER_DAY; + } + + intervalRange = pQueryInfo->interval.interval * f; + } else { + intervalRange = pQueryInfo->interval.interval; + } + // number of result is not greater than 10,000,000 + if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + return TSDB_CODE_SUCCESS; +} + int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0)); const char* msg0 = "invalid table name"; - const char* msg2 = "point interpolation query needs timestamp"; - const char* msg5 = "fill only available for interval query"; - const char* msg6 = "start(end) time of query range required or time range too large"; - const char* msg7 = "illegal number of tables in from clause"; - const char* msg8 = "too many columns in selection clause"; - const char* msg9 = "TWA query requires both the start and end time"; - const char* msg10 = "too many tables in from clause"; - const char* msg11 = "invalid table alias name"; + const char* msg1 = "point interpolation query needs timestamp"; + const char* msg2 = "fill only available for interval query"; + const char* msg3 = "start(end) time of query range required or time range too large"; + const char* msg4 = "illegal number of tables in from clause"; + const char* msg5 = "too many columns in selection clause"; + const char* msg6 = "too many tables in from clause"; + const char* msg7 = "invalid table alias name"; int32_t code = TSDB_CODE_SUCCESS; @@ -6370,7 +6444,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // too many result columns not support order by in query if (pQuerySql->pSelection->nExpr > TSDB_MAX_COLUMNS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } /* @@ -6388,13 +6462,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { size_t fromSize = taosArrayGetSize(pQuerySql->from); if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } pQueryInfo->command = TSDB_SQL_SELECT; if (fromSize > 4) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } // set all query tables, which are maybe more than one. @@ -6427,12 +6501,12 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { tVariantListItem* p1 = taosArrayGet(pQuerySql->from, i + 1); if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING}; if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } // has no table alias name @@ -6510,12 +6584,6 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } } - // user does not specified the query time window, twa is not allowed in such case. - if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX || - (pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - // no result due to invalid query time range if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; @@ -6523,7 +6591,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } if (!hasTimestampForPointInterpQuery(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } // in case of join query, time range is required. @@ -6531,7 +6599,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); if (timeRange == 0 && pQueryInfo->window.skey == 0) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -6545,31 +6613,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { tscFieldInfoUpdateOffset(pQueryInfo); - /* - * fill options are set at the end position, when all columns are set properly - * the columns may be increased due to group by operation - */ if (pQuerySql->fillType != NULL) { if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (pQueryInfo->interval.interval > 0 && pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { - bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); - if (initialWindows) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->interval.interval) > MAX_INTERVAL_TIME_WINDOW) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); - } + /* + * fill options are set at the end position, when all columns are set properly + * the columns may be increased due to group by operation + */ + if ((code = checkQueryRangeForFill(pCmd, pQueryInfo)) != TSDB_CODE_SUCCESS) { + return code; } - int32_t ret = parseFillClause(pCmd, pQueryInfo, pQuerySql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if ((code = parseFillClause(pCmd, pQueryInfo, pQuerySql)) != TSDB_CODE_SUCCESS) { + return code; } } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index fcc93ffadc4f7dc62b79fffe245947799a770b40..b9d38e3ea76d9ba15c7fe5eb5906e205423aade5 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -130,23 +130,15 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { return NULL; } -static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) { - corVgroupInfo->version = 0; - corVgroupInfo->inUse = 0; - corVgroupInfo->numOfEps = vgroupInfo->numOfEps; - for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) { - corVgroupInfo->epAddr[i].fqdn = strdup(vgroupInfo->epAddr[i].fqdn); - corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port; - } -} - -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { - assert(pTableMetaMsg != NULL); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2 && pTableMetaMsg->numOfTags >= 0); int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); + pTableMeta->tableType = pTableMetaMsg->tableType; - + pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, .precision = pTableMetaMsg->precision, @@ -156,22 +148,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->id.tid = pTableMetaMsg->tid; pTableMeta->id.uid = pTableMetaMsg->uid; - SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; - pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps; - pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId; - - for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - SEpAddrMsg* pEpMsg = &pTableMetaMsg->vgroup.epAddr[i]; - - pVgroupInfo->epAddr[i].fqdn = strndup(pEpMsg->fqdn, tListLen(pEpMsg->fqdn)); - pVgroupInfo->epAddr[i].port = pEpMsg->port; - } - - tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo); - pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; - tstrncpy(pTableMeta->sTableId, pTableMetaMsg->sTableId, TSDB_TABLE_FNAME_LEN); + tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN); memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); @@ -180,13 +159,44 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - if (size != NULL) { - *size = sizeof(STableMeta) + schemaSize; - } - return pTableMeta; } +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) { + assert(pExisted != NULL && src != NULL); + if (pExisted->numOfEps != src->numOfEps) { + return false; + } + + for(int32_t i = 0; i < pExisted->numOfEps; ++i) { + if (pExisted->ep[i].port != src->epAddr[i].port) { + return false; + } + + if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) { + return false; + } + } + + return true; +} + +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { + assert(pVgroupMsg != NULL); + + SNewVgroupInfo info = {0}; + info.numOfEps = pVgroupMsg->numOfEps; + info.vgId = pVgroupMsg->vgId; + info.inUse = 0; + + for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) { + tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN); + info.ep[i].port = pVgroupMsg->epAddr[i].port; + } + + return info; +} + // todo refactor UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 960f2561e294c1aad2be6218143166dbd5d9cc46..4949aa9b9dab6efb96f411f433ec420c6fd6d338 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -14,7 +14,6 @@ */ #include "os.h" -#include "tcache.h" #include "tcmdtype.h" #include "trpc.h" #include "tscLocalMerge.h" @@ -45,32 +44,30 @@ static int32_t getWaitingTimeInterval(int32_t count) { return 0; } - return initial * (2<<(count - 2)); + return initial * ((2u)<<(count - 2)); } -static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { - assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); - - SRpcEpSet* pEpSet = &pSql->epSet; +static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupInfo* pVgroupInfo) { + assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); // Issue the query to one of the vnode among a vgroup randomly. // change the inUse property would not affect the isUse attribute of STableMeta pEpSet->inUse = rand() % pVgroupInfo->numOfEps; // apply the FQDN string length check here - bool hasFqdn = false; + bool existed = false; pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; - if (!hasFqdn) { - hasFqdn = (strlen(pEpSet->fqdn[i]) > 0); + int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + if (len > 0) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + existed = true; } } - - assert(hasFqdn); + assert(existed); } static void tscDumpMgmtEpSet(SSqlObj *pSql) { @@ -87,7 +84,8 @@ static void tscEpSetHtons(SRpcEpSet *s) { bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; - } + } + for (int32_t i = 0; i < s1->numOfEps; i++) { if (s1->port[i] != s2->port[i] || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) @@ -95,6 +93,7 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { } return true; } + void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { // no need to update if equal SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; @@ -102,37 +101,45 @@ void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { pCorEpSet->epSet = *pEpSet; taosCorEndWrite(&pCorEpSet->version); } -static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { + +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { if (pVgroupInfo == NULL) { return;} - taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pEpSet->numOfEps = pVgroupInfo->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; } - taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } + + int32_t vgId = pTableMetaInfo->pTableMeta->vgId; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + assert(vgId == 0); + return; + } + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); - taosCorBeginWrite(&pVgroupInfo->version); - tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); - pVgroupInfo->inUse = pEpSet->inUse; - pVgroupInfo->numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - tfree(pVgroupInfo->epAddr[i].fqdn); - pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i])); - pVgroupInfo->epAddr[i].port = pEpSet->port[i]; + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; } - tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); - taosCorEndWrite(&pVgroupInfo->version); + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -147,12 +154,18 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SSqlObj *pSql = tres; SSqlRes *pRes = &pSql->res; - if (code == 0) { + if (code == TSDB_CODE_SUCCESS) { SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; - SRpcEpSet * epSet = &pRsp->epSet; + SRpcEpSet *epSet = &pRsp->epSet; if (epSet->numOfEps > 0) { tscEpSetHtons(epSet); - tscUpdateMgmtEpSet(pSql, epSet); + if (!tscEpSetIsEqual(&pSql->pTscObj->tscCorMgmtEpSet->epSet, epSet)) { + tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); + for (int8_t i = 0; i < epSet->numOfEps; i++) { + tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); + } + tscUpdateMgmtEpSet(pSql, epSet); + } } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -161,11 +174,40 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { tscKillConnection(pObj); return; } else { - if (pRsp->queryId) tscKillQuery(pObj, htonl(pRsp->queryId)); - if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId)); + if (pRsp->queryId) { + tscKillQuery(pObj, htonl(pRsp->queryId)); + } + + if (pRsp->streamId) { + tscKillStream(pObj, htonl(pRsp->streamId)); + } + } + + int32_t total = htonl(pRsp->totalDnodes); + int32_t online = htonl(pRsp->onlineDnodes); + assert(online <= total); + + if (online < total) { + tscError("HB:%p, total dnode:%d, online dnode:%d", pSql, total, online); + pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); } + + pRes->length[0] = total; + pRes->length[1] = online; } else { tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); + if (pRes->length == NULL) { + pRes->length = calloc(2, sizeof(int32_t)); + } + + pRes->length[1] = 0; + if (pRes->length[0] == 0) { + pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node + } } if (pObj->hbrid != 0) { @@ -269,7 +311,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { return; } - if (pEpSet) { + if (pEpSet) { // todo update this if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if (pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); @@ -291,7 +333,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { - tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); + + pSql->retry++; + tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), pSql->retry); pSql->res.code = rpcMsg->code; // keep the previous error code if (pSql->retry > pSql->maxRetry) { @@ -401,7 +445,7 @@ int doProcessSql(SSqlObj *pSql) { } if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return pRes->code; } @@ -410,7 +454,7 @@ int doProcessSql(SSqlObj *pSql) { // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. if (code != TSDB_CODE_SUCCESS) { pRes->code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; } @@ -478,8 +522,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); - tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId); + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); + tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgId); } pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); @@ -498,7 +542,6 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // NOTE: shell message size should not include SMsgDesc int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); - int32_t vgId = pTableMeta->vgroupInfo.vgId; SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; pMsgDesc->numOfVnodes = htonl(1); // always one vnode @@ -506,7 +549,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(SMsgDesc); SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; - pShellMsg->header.vgId = htonl(vgId); + pShellMsg->header.vgId = htonl(pTableMeta->vgId); pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc pShellMsg->length = pShellMsg->header.contLen; @@ -514,9 +557,12 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epSet); - tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, pTableMeta->vgId, pSql->cmd.numOfTablesInSubmit, pSql->epSet.numOfEps); return TSDB_CODE_SUCCESS; } @@ -524,9 +570,11 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { /* * for table query, simply return the size <= 1k */ -static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); @@ -534,6 +582,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs * 2); int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + int32_t tableSerialize = 0; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -550,7 +600,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + - tableSerialize + 4096; + tableSerialize + sqlLen + 4096; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { @@ -560,24 +610,31 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - SVgroupInfo* pVgroupInfo = NULL; + int32_t vgId = -1; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t index = pTableMetaInfo->vgroupIndex; assert(index >= 0); - + + SVgroupInfo* pVgroupInfo = NULL; if (pTableMetaInfo->vgroupList->numOfVgroups > 0) { assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; } + + vgId = pVgroupInfo->vgId; + tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); } else { - pVgroupInfo = &pTableMeta->vgroupInfo; + vgId = pTableMeta->vgId; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); } - assert(pVgroupInfo != NULL); + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - tscSetDnodeEpSet(pSql, pVgroupInfo); - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); + pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->id.tid); @@ -596,7 +653,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo); + tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); @@ -623,7 +680,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - int32_t size = tscEstimateQueryMsgSize(pCmd, pCmd->clauseIndex); + int32_t size = tscEstimateQueryMsgSize(pSql, pCmd->clauseIndex); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); @@ -656,7 +713,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList); - + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); @@ -679,10 +737,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); - + pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); + pQueryMsg->sqlstrLen = htonl(sqlLen); + size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number @@ -851,13 +911,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); - *((int16_t *)pMsg) = pCol->colId; + *((int16_t *)pMsg) = htons(pCol->colId); pMsg += sizeof(pCol->colId); - *((int16_t *)pMsg) += pCol->colIndex; + *((int16_t *)pMsg) += htons(pCol->colIndex); pMsg += sizeof(pCol->colIndex); - *((int16_t *)pMsg) += pCol->flag; + *((int16_t *)pMsg) += htons(pCol->flag); pMsg += sizeof(pCol->flag); memcpy(pMsg, pCol->name, tListLen(pCol->name)); @@ -915,13 +975,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += pCond->len; } } - - if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { - *pMsg = 0; - pMsg++; - } else { - strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); - pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; } // compressed ts block @@ -942,6 +1000,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); } + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); tscDebug("%p msg built success, len:%d bytes", pSql, msgLen); @@ -1210,7 +1271,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pEpAddr->n); } - pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen; + pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); return TSDB_CODE_SUCCESS; } @@ -1234,12 +1295,12 @@ int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &(pSql->cmd); - - int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg); + int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo; if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { - size += sizeof(STagData); + int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); + size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); } else { size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); } @@ -1267,33 +1328,55 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; - strcpy(pCreateTableMsg->tableId, pTableMetaInfo->name); - // use dbinfo from table id without modifying current db info - tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateTableMsg->db); + SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); + char* pMsg = NULL; + + int8_t type = pInfo->pCreateTableInfo->type; + if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value + SArray* list = pInfo->pCreateTableInfo->childTableInfo; - SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo; + int32_t numOfTables = (int32_t) taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); - pCreateTableMsg->igExists = pCreateTable->existCheck ? 1 : 0; - pCreateTableMsg->numOfColumns = htons(pCmd->numOfCols); - pCreateTableMsg->numOfTags = htons(pCmd->count); + pMsg = (char*) pCreateMsg; + for(int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; - pCreateTableMsg->sqlLen = 0; - char *pMsg = (char *)pCreateTableMsg->schema; + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); - int8_t type = pInfo->pCreateTableInfo->type; - if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value - STagData* pTag = &pInfo->pCreateTableInfo->usingInfo.tagdata; - *(int32_t*)pMsg = htonl(pTag->dataLen); - pMsg += sizeof(int32_t); - memcpy(pMsg, pTag->name, sizeof(pTag->name)); - pMsg += sizeof(pTag->name); - memcpy(pMsg, pTag->data, pTag->dataLen); - pMsg += pTag->dataLen; + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableId, p->fullname); + pCreate->igExists = (p->igExist)? 1 : 0; + + // use dbinfo from table id without modifying current db info + tscGetDBInfoFromTableFullName(p->fullname, pCreate->db); + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*) pCreate); + pCreate->len = htonl(len); + } } else { // create (super) table - pSchema = (SSchema *)pCreateTableMsg->schema; + pCreateTableMsg->numOfTables = htonl(1); // only one table will be created + + strcpy(pCreateMsg->tableId, pTableMetaInfo->name); + + // use dbinfo from table id without modifying current db info + tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateMsg->db); + + SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo; + + pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; + pCreateMsg->numOfColumns = htons(pCmd->numOfCols); + pCreateMsg->numOfTags = htons(pCmd->count); + + pCreateMsg->sqlLen = 0; + pMsg = (char *)pCreateMsg->schema; + + pSchema = (SSchema *)pCreateMsg->schema; for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); @@ -1310,7 +1393,7 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQuerySQL *pQuerySql = pInfo->pCreateTableInfo->pSelect; strncpy(pMsg, pQuerySql->selectToken.z, pQuerySql->selectToken.n + 1); - pCreateTableMsg->sqlLen = htons(pQuerySql->selectToken.n + 1); + pCreateMsg->sqlLen = htons(pQuerySql->selectToken.n + 1); pMsg += pQuerySql->selectToken.n + 1; } } @@ -1386,51 +1469,18 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.vgId > 0); - tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epSet); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); return TSDB_CODE_SUCCESS; } -//int tscBuildCancelQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { -// SCancelQueryMsg *pCancelMsg = (SCancelQueryMsg*) pSql->cmd.payload; -// pCancelMsg->qhandle = htobe64(pSql->res.qhandle); -// -// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); -// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// -// if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { -// int32_t vgIndex = pTableMetaInfo->vgroupIndex; -// if (pTableMetaInfo->pVgroupTables == NULL) { -// SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; -// assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); -// -// pCancelMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); -// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex); -// } else { -// int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); -// assert(vgIndex >= 0 && vgIndex < numOfVgroups); -// -// SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); -// -// pCancelMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId); -// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex); -// } -// } else { -// STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; -// pCancelMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); -// tscDebug("%p build cancel query msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId); -// } -// -// pSql->cmd.payloadLen = sizeof(SCancelQueryMsg); -// pSql->cmd.msgType = TSDB_MSG_TYPE_CANCEL_QUERY; -// -// pCancelMsg->header.contLen = htonl(sizeof(SCancelQueryMsg)); -// return TSDB_CODE_SUCCESS; -//} - int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SAlterDbMsg); @@ -1492,7 +1542,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { if (code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -1521,7 +1571,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { int32_t code = pRes->code; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; } @@ -1540,7 +1590,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; @@ -1587,13 +1637,8 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - if (pCmd->autoCreated && pCmd->pTagData != NULL) { - int len = htonl(pCmd->pTagData->dataLen); - if (len > 0) { - len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen); - memcpy(pInfoMsg->tags, pCmd->pTagData, len); - pMsg += len; - } + if (pCmd->autoCreated && pCmd->tagData.dataLen != 0) { + pMsg = serializeTagData(&pCmd->tagData, pMsg); } pCmd->payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); @@ -1789,19 +1834,46 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pSchema++; } - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - // todo add one more function: taosAddDataIfNotExists(); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // check if super table hashmap or not + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - if (pTableMetaInfo->pTableMeta == NULL) { - free(pTableMeta); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + // super tableMeta data alreay exists, create it according to tableMeta and add it to hash map + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, s); + } + + // update the vgroupInfo if needed + if (pTableMeta->vgId > 0) { + int32_t vgId = pTableMeta->vgId; + assert(pTableMeta->tableType != TSDB_SUPER_TABLE); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, &pMetaMsg->vgroup)) || + (vgroupInfo.inUse < 0)) { // vgroup info exists, compare with it + vgroupInfo = createNewVgroupInfo(&pMetaMsg->vgroup); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add new VgroupInfo, vgId:%d, total:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } } tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name); @@ -1812,8 +1884,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { /** * multi table meta rsp pkg format: - * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 - * |...... 1B 1B 4B + * | STaosRsp | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * |...... 1B 4B **/ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { #if 0 @@ -1967,14 +2039,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { return pSql->res.code; } -/* - * current process do not use the cache at all - */ int tscProcessShowRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg; SShowRsp * pShow; SSchema * pSchema; - char key[20]; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -1999,20 +2067,10 @@ int tscProcessShowRsp(SSqlObj *pSql) { pSchema++; } - key[0] = pCmd->msgType + 'a'; - strcpy(key + 1, "showlist"); - - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false); - } + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - - pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } @@ -2035,12 +2093,9 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); - - tfree(pTableMeta); return 0; } -// TODO multithread problem static void createHBObj(STscObj* pObj) { if (pObj->hbrid != 0) { return; @@ -2093,6 +2148,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) { if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); tscUpdateMgmtEpSet(pSql, &pConnect->epSet); + + for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { + tscDebug("%p epSet.fqdn[%d]: %s, pObj:%p", pSql, i, pConnect->epSet.fqdn[i], pObj); + } } strcpy(pObj->sversion, pConnect->serverVersion); @@ -2118,54 +2177,34 @@ int tscProcessUseDbRsp(SSqlObj *pSql) { int tscProcessDropDbRsp(SSqlObj *pSql) { pSql->pTscObj->db[0] = 0; - taosCacheEmpty(tscMetaCache); + taosHashEmpty(tscTableMetaInfo); return 0; } int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } - - /* - * 1. if a user drops one table, which is the only table in a vnode, remove operation will incur vnode to be removed. - * 2. Then, a user creates a new metric followed by a table with identical name of removed table but different schema, - * here the table will reside in a new vnode. - * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache - * instead. - */ - tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); - - if (pTableMetaInfo->pTableMeta) { - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); - } + //The cached tableMeta is expired in this case, so clean it in hash table + taosHashRemove(tscTableMetaInfo, pTableMetaInfo->name, strnlen(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN)); + tscDebug("%p remove table meta after drop table:%s, numOfRemain:%d", pSql, pTableMetaInfo->name, + (int32_t) taosHashGetSize(tscTableMetaInfo)); + assert(pTableMetaInfo->pTableMeta == NULL); return 0; } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } - - tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); + char* name = pTableMetaInfo->name; + tscDebug("%p remove tableMeta in hashMap after alter-table: %s", pSql, name); - if (pTableMetaInfo->pTableMeta) { - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); - if (isSuperTable) { // if it is a super table, reset whole query cache - tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); - taosCacheEmpty(tscMetaCache); - } + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + taosHashEmpty(tscTableMetaInfo); } return 0; @@ -2175,6 +2214,7 @@ int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { UNUSED(pSql); return 0; } + int tscProcessShowCreateRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 1); } @@ -2250,7 +2290,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); -static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { +static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); if (NULL == pNew) { tscError("%p malloc failed for new sqlobj to get table meta", pSql); @@ -2277,15 +2317,13 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); - if (pSql->cmd.pTagData != NULL) { - int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); - pNew->cmd.pTagData = calloc(1, size); - if (pNew->cmd.pTagData == NULL) { + if (pSql->cmd.autoCreated) { + int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); + if (code != TSDB_CODE_SUCCESS) { tscError("%p malloc failed for new tag data to get table meta", pSql); tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); } tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); @@ -2297,7 +2335,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated } return code; @@ -2305,31 +2343,39 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { assert(strlen(pTableMetaInfo->name) != 0); + tfree(pTableMetaInfo->pTableMeta); - // If this STableMetaInfo owns a table meta, release it first - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false); - } - - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMetaInfo->pTableMeta != NULL) { - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, - tinfo.numOfTags, pTableMetaInfo->pTableMeta); + uint32_t size = tscGetTableMetaMaxSize(); + pTableMetaInfo->pTableMeta = calloc(1, size); + + pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; + int32_t len = (int32_t) strlen(pTableMetaInfo->name); + + taosHashGetClone(tscTableMetaInfo, pTableMetaInfo->name, len, NULL, pTableMetaInfo->pTableMeta, -1); + + // TODO resize the tableMeta + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + if (pMeta->id.uid > 0) { + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, pTableMetaInfo->name); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo); + } + } return TSDB_CODE_SUCCESS; } - - return getTableMetaFromMgmt(pSql, pTableMetaInfo); + + return getTableMetaFromMnode(pSql, pTableMetaInfo); } -int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { +int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists) { pSql->cmd.autoCreated = createIfNotExists; return tscGetTableMeta(pSql, pTableMetaInfo); } /** - * retrieve table meta from mnode, and update the local table meta cache. + * retrieve table meta from mnode, and update the local table meta hashmap. * @param pSql sql object * @param tableIndex table index * @return status code @@ -2337,17 +2383,19 @@ int tscGetMeterMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + const char* name = pTableMetaInfo->name; STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pTableMeta) { - tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta); + if (pTableMeta) { + tscDebug("%p update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64, pSql, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); } - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); - return getTableMetaFromMgmt(pSql, pTableMetaInfo); + // remove stored tableMeta info in hash table + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + return getTableMetaFromMnode(pSql, pTableMetaInfo); } static bool allVgroupInfoRetrieved(SSqlCmd* pCmd, int32_t clauseIndex) { @@ -2387,7 +2435,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); + STableMeta* pTableMeta = tscTableMetaClone(pMInfo->pTableMeta); tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d7dec2f35643dfb68e66e3d8197f3a735d5a700e..377cb24b1d5b1c92c702cd113c4feea185617911 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -115,9 +115,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa pObj->signature = pObj; pObj->pDnodeConn = pDnodeConn; - T_REF_INIT_VAL(pObj, 1); - tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); memcpy(pObj->pass, secretEncrypt, secretEncryptLen); @@ -172,11 +170,9 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (taos != NULL) { *taos = pObj; } - + pObj->rid = taosAddRef(tscRefId, pObj); registerSqlObj(pSql); - tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - pObj->rid = taosAddRef(tscRefId, pObj); return pSql; } @@ -288,34 +284,21 @@ void taos_close(TAOS *taos) { return; } - // make sure that the close connection can only be executed once. - pObj->signature = NULL; - taosTmrStopA(&(pObj->pTimer)); - - if (pObj->hbrid > 0) { + if (RID_VALID(pObj->hbrid)) { SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); if (pHb != NULL) { - if (pHb->rpcRid > 0) { // wait for rsp from dnode + if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode rpcCancelRequest(pHb->rpcRid); pHb->rpcRid = -1; } tscDebug("%p HB is freed", pHb); - taos_free_result(pHb); taosReleaseRef(tscObjRef, pHb->self); + taos_free_result(pHb); } } - int32_t ref = T_REF_DEC(pObj); - assert(ref >= 0); - - if (ref > 0) { - tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn); - return; - } - tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); - taosRemoveRef(tscRefId, pObj->rid); } @@ -331,7 +314,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { tsem_post(&pSql->rspSem); } -TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) { +TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, int64_t* res) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { terrno = TSDB_CODE_TSC_DISCONNECTED; @@ -357,7 +340,7 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); if (res != NULL) { - *res = pSql; + atomic_store_64(res, pSql->self); } tsem_wait(&pSql->rspSem); @@ -368,7 +351,7 @@ TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL); } -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) { +TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res) { return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res); } @@ -726,7 +709,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) { pSubObj->rpcRid = -1; } - tscQueueAsyncRes(pSubObj); + tscAsyncResultOnError(pSubObj); taosReleaseRef(tscObjRef, pSubObj->self); } @@ -762,7 +745,7 @@ void taos_stop_query(TAOS_RES *res) { pSql->rpcRid = -1; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -926,7 +909,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) { // must before clean the sqlcmd object - tscResetSqlCmdObj(&pSql->cmd, false); + tscResetSqlCmdObj(&pSql->cmd); SSqlCmd *pCmd = &pSql->cmd; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 74b8e4d95878d8a784d4d82e28028e94bd32207b..c1ed9b0ba09f956207eeeccb2c7b8123538a2151 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -167,7 +167,9 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); + + char* name = pTableMetaInfo->name; + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); @@ -269,9 +271,8 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name, pStream->numOfRes); - // release the metric/meter meta information reference, so data in cache can be updated + tfree(pTableMetaInfo->pTableMeta); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 52b74f7502db139f18d8230eefb36d41bae4079f..7f0b174ad364c783d03b3e7c7ee9bf8854c27cc7 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -69,14 +69,17 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { } void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { - if( sub == NULL) + if( sub == NULL) { return; + } + SSub* pSub = (SSub*)sub; SSubscriptionProgress target = {.uid = uid, .key = ts}; SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); if (p != NULL) { p->key = ts; + tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key); } } @@ -502,6 +505,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key; + tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); } if (pSub->pTimer == NULL) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d2eb16795f88356633dc96768379fe058426ef12..681291d0db7039af09147d86f6db1fa48ccb233c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -384,7 +384,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { } SQueryInfo *pSubQueryInfo = tscGetQueryInfoDetail(&pPrevSub->cmd, 0); - STSBuf *pTSBuf = pSubQueryInfo->tsBuf; + STSBuf *pTsBuf = pSubQueryInfo->tsBuf; pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj @@ -402,7 +402,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pSql->pSubs[i] = pNew; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - pQueryInfo->tsBuf = pTSBuf; // transfer the ownership of timestamp comp-z data to the new created object + pQueryInfo->tsBuf = pTsBuf; // transfer the ownership of timestamp comp-z data to the new created object // set the second stage sub query for join process TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); @@ -779,7 +779,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = numOfRows; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -796,7 +796,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -845,7 +845,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (code != TSDB_CODE_SUCCESS) { freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); taosArrayDestroy(s1); taosArrayDestroy(s2); @@ -916,7 +916,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = numOfRows; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -930,7 +930,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1028,7 +1028,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pParentSql->res.code = numOfRows; tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows)); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1155,7 +1155,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1233,7 +1233,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1344,7 +1344,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pParentSql->res.code != TSDB_CODE_SUCCESS) { tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code); quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1357,7 +1357,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { pParentSql->res.code = code; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1403,7 +1403,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1612,7 +1612,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { _error: pRes->code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { @@ -1648,7 +1648,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->qhandle = 0x1; // hack the qhandle check - const uint32_t nBufferSize = (1u << 16); // 64KB + const uint32_t nBufferSize = (1u << 16u); // 64KB SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -1666,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, &pFinalModel, nBufferSize); if (ret != 0) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); tfree(pMemoryBuf); return ret; } @@ -1680,7 +1680,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel,pState->numOfSub); - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return ret; } @@ -1890,7 +1890,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); } else { // regular super table query if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1968,7 +1968,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } @@ -2151,7 +2151,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { static bool needRetryInsert(SSqlObj* pParentObj, int32_t numOfSub) { if (pParentObj->retry > pParentObj->maxRetry) { - tscError("%p max retry reached, abort the retry effort", pParentObj) + tscError("%p max retry reached, abort the retry effort", pParentObj); return false; } @@ -2209,10 +2209,10 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) // restore user defined fp pParentObj->fp = pParentObj->fetchFp; int32_t numOfSub = pParentObj->subState.numOfSub; + doFreeInsertSupporter(pParentObj); if (pParentObj->res.code == TSDB_CODE_SUCCESS) { tscDebug("%p Async insertion completed, total inserted:%d", pParentObj, pParentObj->res.numOfRows); - doFreeInsertSupporter(pParentObj); // todo remove this parameter in async callback function definition. // all data has been sent to vnode, call user function @@ -2220,8 +2220,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) (*pParentObj->fp)(pParentObj->param, pParentObj, v); } else { if (!needRetryInsert(pParentObj, numOfSub)) { - doFreeInsertSupporter(pParentObj); - tscQueueAsyncRes(pParentObj); + tscAsyncResultOnError(pParentObj); return; } @@ -2232,7 +2231,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) numOfFailed += 1; // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd, true); + tscFreeQueryInfo(&pSql->cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); @@ -2244,27 +2243,29 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) tscError("%p Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj, pParentObj->res.numOfRows, numOfFailed, numOfSub); - tscDebug("%p cleanup %d tableMeta in cache", pParentObj, pParentObj->cmd.numOfTables); + tscDebug("%p cleanup %d tableMeta in hashTable", pParentObj, pParentObj->cmd.numOfTables); for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) { - taosCacheRelease(tscMetaCache, (void**)&(pParentObj->cmd.pTableMetaList[i]), true); + char* name = pParentObj->cmd.pTableNameList[i]; + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } pParentObj->cmd.parseFinished = false; pParentObj->subState.numOfRemain = numOfFailed; - tscResetSqlCmdObj(&pParentObj->cmd, false); + tscResetSqlCmdObj(&pParentObj->cmd); // in case of insert, redo parsing the sql string and build new submit data block for two reasons: // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. // 2. vnode may need the schema information along with submit block to update its local table schema. - tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry++); + tscDebug("%p re-parse sql to generate submit data, retry:%d", pParentObj, pParentObj->retry); + pParentObj->retry++; + int32_t code = tsParseSql(pParentObj, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return; if (code != TSDB_CODE_SUCCESS) { pParentObj->res.code = code; - doFreeInsertSupporter(pParentObj); - tscQueueAsyncRes(pParentObj); + tscAsyncResultOnError(pParentObj); return; } @@ -2279,7 +2280,6 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) */ int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { assert(pSql != NULL && pSql->param != NULL); -// SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; SInsertSupporter* pSupporter = (SInsertSupporter*) pSql->param; @@ -2288,11 +2288,8 @@ int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { STableDataBlocks* pTableDataBlock = taosArrayGetP(pParent->cmd.pDataBlocks, pSupporter->index); int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); - // free the data block created from insert sql string -// pCmd->pDataBlocks = tscDestroyBlockArrayList(pParent->cmd.pDataBlocks); - if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; // here the pSql may have been released already. } @@ -2307,7 +2304,11 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { if (pSql->pSubs != NULL) { for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; + SInsertSupporter* pSup = calloc(1, sizeof(SInsertSupporter)); + pSup->index = i; + pSup->pSql = pSql; + pSub->param = pSup; tscDebug("%p sub:%p launch sub insert, orderOfSub:%d", pSql, pSub, i); if (pSub->res.code != TSDB_CODE_SUCCESS) { tscHandleInsertRetry(pSql, pSub); @@ -2481,7 +2482,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SSqlRes* pRes = &pSql->res; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -2496,19 +2497,19 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } tscRestoreSQLFuncForSTableQuery(pQueryInfo); } - while (1) { - assert (pRes->row >= pRes->numOfRows); - - doBuildResFromSubqueries(pSql); - tsem_post(&pSql->rspSem); - return; + assert (pRes->row >= pRes->numOfRows); + doBuildResFromSubqueries(pSql); + if (pRes->code == TSDB_CODE_SUCCESS) { + (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); + } else { + tscAsyncResultOnError(pSql); } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 1eddeacc6514356d7a45874c33e9132443a83eca..dd0f248b85452342fb567357ba9346ab71754dfc 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -31,18 +31,21 @@ #include "tlocale.h" // global, not configurable -SCacheObj* tscMetaCache; -int tscObjRef = -1; -void * tscTmr; -void * tscQhandle; -void * tscCheckDiskUsageTmr; -int tsInsertHeadSize; -int tscRefId = -1; +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 -int tscNumOfThreads; +int32_t sentinel = TSC_VAR_NOT_RELEASE; +SHashObj *tscVgroupMap; // hash map to keep the global vgroup info +SHashObj *tscTableMetaInfo; // table meta info +int32_t tscObjRef = -1; +void *tscTmr; +void *tscQhandle; +int32_t tscRefId = -1; +int32_t tscNumOfObj = 0; // number of sqlObj in current process. + +static void *tscCheckDiskUsageTmr; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -//void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet); void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); @@ -115,7 +118,7 @@ void taos_init_imp(void) { int queueSize = tsMaxConnections*2; double factor = (tscEmbedded == 0)? 2.0:4.0; - tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); + int32_t tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); if (tscNumOfThreads < 2) { tscNumOfThreads = 2; } @@ -131,10 +134,11 @@ void taos_init_imp(void) { taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); } - int64_t refreshTime = 10; // 10 seconds by default - if (tscMetaCache == NULL) { - tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); - tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + if (tscTableMetaInfo == NULL) { + tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + tscTableMetaInfo = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tscDebug("TableMeta:%p", tscTableMetaInfo); } tscRefId = taosOpenRef(200, tscCloseTscObj); @@ -152,30 +156,38 @@ void taos_init() { pthread_once(&tscinit, taos_init_imp); } void taos_cleanup(void) { tscDebug("start to cleanup client environment"); - void* m = tscMetaCache; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) { - taosCacheCleanup(m); + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; } - int refId = atomic_exchange_32(&tscObjRef, -1); - if (refId != -1) { - taosCloseRef(refId); - } + taosHashCleanup(tscTableMetaInfo); + tscTableMetaInfo = NULL; - m = tscQhandle; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) { - taosCleanUpScheduler(m); - } + taosHashCleanup(tscVgroupMap); + tscVgroupMap = NULL; + + int32_t id = tscObjRef; + tscObjRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscRefId; + tscRefId = -1; + taosCloseRef(id); - taosCloseRef(tscRefId); taosCleanupKeywordsTable(); taosCloseLog(); - if (tscEmbedded == 0) rpcCleanup(); - m = tscTmr; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) { - taosTmrCleanUp(m); + if (tscEmbedded == 0) { + rpcCleanup(); } + + p = tscTmr; + tscTmr = NULL; + taosTmrCleanUp(p); } static int taos_options_imp(TSDB_OPTION option, const char *pStr) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index dbd626d360fa6948396753bb7c76da7e5f8dcf23..a6e33778cde0a43f4cdd003e718ef49b89a41866 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -18,7 +18,6 @@ #include "os.h" #include "qAst.h" #include "taosmsg.h" -#include "tcache.h" #include "tkey.h" #include "tmd5.h" #include "tscLocalMerge.h" @@ -31,7 +30,7 @@ #include "ttokendef.h" static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); -static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); +static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo); SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { @@ -379,17 +378,16 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { +void tscFreeQueryInfo(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } for (int32_t i = 0; i < pCmd->numOfClause; ++i) { - char* addr = (char*)pCmd - offsetof(SSqlObj, cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache); + clearAllTableMetaInfo(pQueryInfo); tfree(pQueryInfo); } @@ -397,7 +395,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { tfree(pCmd->pQueryInfo); } -void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { +void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; @@ -405,13 +403,19 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->msgType = 0; pCmd->parseFinished = 0; pCmd->autoCreated = 0; - pCmd->numOfTables = 0; - tfree(pCmd->pTableMetaList); + for(int32_t i = 0; i < pCmd->numOfTables; ++i) { + if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { + tfree(pCmd->pTableNameList[i]); + } + } + + pCmd->numOfTables = 0; + tfree(pCmd->pTableNameList); pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd, removeFromCache); + tscFreeQueryInfo(pCmd); } void tscFreeSqlResult(SSqlObj* pSql) { @@ -452,35 +456,14 @@ void tscFreeRegisteredSqlObj(void *pSql) { SSqlObj* p = *(SSqlObj**)pSql; STscObj* pTscObj = p->pTscObj; - assert(p->self != 0); - tscFreeSqlObj(p); - - int32_t ref = T_REF_DEC(pTscObj); - assert(ref >= 0); - - tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref); - if (ref == 0) { - tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj); - taosRemoveRef(tscRefId, pTscObj->rid); - } -} - -void tscFreeTableMetaHelper(void *pTableMeta) { - STableMeta* p = (STableMeta*) pTableMeta; - - int32_t numOfEps = p->vgroupInfo.numOfEps; - assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA); + assert(RID_VALID(p->self)); - for(int32_t i = 0; i < numOfEps; ++i) { - tfree(p->vgroupInfo.epAddr[i].fqdn); - } - - int32_t numOfEps1 = p->corVgroupInfo.numOfEps; - assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA); + tscFreeSqlObj(p); + taosReleaseRef(tscRefId, pTscObj->rid); - for(int32_t i = 0; i < numOfEps1; ++i) { - tfree(p->corVgroupInfo.epAddr[i].fqdn); - } + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfObj, 1); + int32_t total = atomic_sub_fetch_32(&tscNumOfObj, 1); + tscDebug("%p free SqlObj, total in tscObj:%d, total:%d", pSql, num, total); } void tscFreeSqlObj(SSqlObj* pSql) { @@ -510,9 +493,10 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->self = 0; tscFreeSqlResult(pSql); - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmdObj(pCmd); - tfree(pCmd->pTagData); + tfree(pCmd->tagData.data); + pCmd->tagData.dataLen = 0; memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); @@ -532,7 +516,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); + tfree(pDataBlock->pTableMeta); } tfree(pDataBlock); @@ -603,15 +587,15 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { - tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name)); + tstrncpy(pTableMetaInfo->name, pDataBlock->tableName, sizeof(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); + tfree(pTableMetaInfo->pTableMeta); } - pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta); + pTableMetaInfo->pTableMeta = tscTableMetaClone(pDataBlock->pTableMeta); } else { - assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); + assert(strncmp(pTableMetaInfo->name, pDataBlock->tableName, tListLen(pDataBlock->tableName)) == 0); } /* @@ -674,14 +658,10 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff dataBuf->size = startOffset; dataBuf->tsSource = -1; - tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId)); + tstrncpy(dataBuf->tableName, name, sizeof(dataBuf->tableName)); - /* - * The table meta may be released since the table meta cache are completed clean by other thread - * due to operation such as drop database. So here we add the reference count directly instead of invoke - * taosGetDataFromCache, which may return NULL value. - */ - dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta); + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tscTableMetaClone(pTableMeta); assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; @@ -787,15 +767,15 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { return result; } -static void extractTableMeta(SSqlCmd* pCmd) { +static void extractTableNameList(SSqlCmd* pCmd) { pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList); - pCmd->pTableMetaList = calloc(pCmd->numOfTables, POINTER_BYTES); + pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES); STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL); int32_t i = 0; while(p1) { STableDataBlocks* pBlocks = *p1; - pCmd->pTableMetaList[i++] = taosCacheTransfer(tscMetaCache, (void**) &pBlocks->pTableMeta); + pCmd->pTableNameList[i++] = strndup(pBlocks->tableName, TSDB_TABLE_FNAME_LEN); p1 = taosHashIterate(pCmd->pTableBlockHashList, p1); } @@ -803,6 +783,7 @@ static void extractTableMeta(SSqlCmd* pCmd) { } int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { + const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); SSqlCmd* pCmd = &pSql->cmd; void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); @@ -817,7 +798,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { STableDataBlocks* dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); taosHashCleanup(pVnodeDataBlockHashList); @@ -851,7 +832,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { tscSortRemoveDataBlockDupRows(pOneTableBlock); char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - tscDebug("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, + tscDebug("%p name:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableName, pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); @@ -881,7 +862,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { pOneTableBlock = *p; } - extractTableMeta(pCmd); + extractTableNameList(pCmd); // free the table data blocks; pCmd->pDataBlocks = pVnodeDataBlockList; @@ -902,6 +883,7 @@ void tscCloseTscObj(void *param) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; } + tfree(pObj->tscCorMgmtEpSet); pthread_mutex_destroy(&pObj->mutex); @@ -1530,6 +1512,7 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { } dest->tbnameCond.uid = src->tbnameCond.uid; + dest->tbnameCond.len = src->tbnameCond.len; memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); dest->relType = src->relType; @@ -1825,14 +1808,12 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { return pa; } -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables); - +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); + tscClearTableMetaInfo(pTableMetaInfo); free(pTableMetaInfo); } @@ -1886,14 +1867,12 @@ STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); } -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) { if (pTableMetaInfo == NULL) { return; } - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - } + tfree(pTableMetaInfo->pTableMeta); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); @@ -1910,10 +1889,12 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { } void registerSqlObj(SSqlObj* pSql) { - int32_t ref = T_REF_INC(pSql->pTscObj); - tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref); - + taosAcquireRef(tscRefId, pSql->pTscObj->rid); pSql->self = taosAddRef(tscObjRef, pSql); + + int32_t num = atomic_add_fetch_32(&pSql->pTscObj->numOfObj, 1); + int32_t total = atomic_add_fetch_32(&tscNumOfObj, 1); + tscDebug("%p new SqlObj from %p, total in tscObj:%d, total:%d", pSql, pSql->pTscObj, num, total); } SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) { @@ -1931,15 +1912,11 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pCmd->parseFinished = 1; pCmd->autoCreated = pSql->cmd.autoCreated; - if (pSql->cmd.pTagData != NULL) { - int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); - pNew->cmd.pTagData = calloc(1, size); - if (pNew->cmd.pTagData == NULL) { - tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); - free(pNew); - return NULL; - } - memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); + if (code != TSDB_CODE_SUCCESS) { + tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); + free(pNew); + return NULL; } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { @@ -1947,30 +1924,24 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return NULL; } - pNew->fp = fp; + pNew->fp = fp; pNew->fetchFp = fp; - pNew->param = param; + pNew->param = param; + pNew->sqlstr = NULL; pNew->maxRetry = TSDB_MAX_REPLICA; - pNew->sqlstr = strdup(pSql->sqlstr); - if (pNew->sqlstr == NULL) { - tscError("%p new subquery failed", pSql); - tscFreeSqlObj(pNew); - return NULL; - } - SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); - registerSqlObj(pNew); + return pNew; } -static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) { +static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t uid) { int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo); if (numOfOutput == 0) { return; @@ -2023,15 +1994,9 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); - pNew->pTscObj = pSql->pTscObj; + pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; - - pNew->sqlstr = strdup(pSql->sqlstr); - if (pNew->sqlstr == NULL) { - tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } + pNew->sqlstr = strdup(pSql->sqlstr); SSqlCmd* pnCmd = &pNew->cmd; memcpy(pnCmd, pCmd, sizeof(SSqlCmd)); @@ -2044,7 +2009,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->numOfClause = 0; pnCmd->clauseIndex = 0; pnCmd->pDataBlocks = NULL; + + pnCmd->numOfTables = 0; pnCmd->parseFinished = 1; + pnCmd->pTableNameList = NULL; + pnCmd->pTableBlockHashList = NULL; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -2115,35 +2084,34 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void goto _error; } - doSetSqlExprAndResultFieldInfo(pQueryInfo, pNewQueryInfo, uid); + doSetSqlExprAndResultFieldInfo(pNewQueryInfo, uid); - pNew->fp = fp; + pNew->fp = fp; pNew->fetchFp = fp; - - pNew->param = param; + pNew->param = param; pNew->maxRetry = TSDB_MAX_REPLICA; char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { - STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup + STableMeta* pTableMeta = tscTableMetaClone(pTableMetaInfo->pTableMeta); assert(pTableMeta != NULL); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, - pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); + pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); - + STableMeta* pPrevTableMeta = tscTableMetaClone(pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } + // this case cannot be happened if (pFinalInfo->pTableMeta == NULL) { - tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); + tscError("%p new subquery failed since no tableMeta, name:%s", pSql, name); if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); @@ -2516,7 +2484,7 @@ bool tscSetSqlOwner(SSqlObj* pSql) { SSqlRes* pRes = &pSql->res; // set the sql object owner - uint64_t threadId = taosGetPthreadId(); + uint64_t threadId = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; @@ -2570,6 +2538,10 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { tfree(pVgroupInfo->epAddr[j].fqdn); } + + for(int32_t j = pVgroupInfo->numOfEps; j < TSDB_MAX_REPLICA; j++) { + assert( pVgroupInfo->epAddr[j].fqdn == NULL ); + } } tfree(vgroupList); @@ -2582,6 +2554,125 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) { for(int32_t i = 0; i < dst->numOfEps; ++i) { tfree(dst->epAddr[i].fqdn); dst->epAddr[i].port = src->epAddr[i].port; + assert(dst->epAddr[i].fqdn == NULL); + dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); } } + +char* serializeTagData(STagData* pTagData, char* pMsg) { + int32_t n = (int32_t) strlen(pTagData->name); + *(int32_t*) pMsg = htonl(n); + pMsg += sizeof(n); + + memcpy(pMsg, pTagData->name, n); + pMsg += n; + + *(int32_t*)pMsg = htonl(pTagData->dataLen); + pMsg += sizeof(int32_t); + + memcpy(pMsg, pTagData->data, pTagData->dataLen); + pMsg += pTagData->dataLen; + + return pMsg; +} + +int32_t copyTagData(STagData* dst, const STagData* src) { + dst->dataLen = src->dataLen; + tstrncpy(dst->name, src->name, tListLen(dst->name)); + + if (dst->dataLen > 0) { + dst->data = malloc(dst->dataLen); + if (dst->data == NULL) { + return -1; + } + + memcpy(dst->data, src->data, dst->dataLen); + } + + return 0; +} + +STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { + assert(pChild != NULL); + int32_t total = pChild->numOfColumns + pChild->numOfTags; + + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); + pTableMeta->tableType = TSDB_SUPER_TABLE; + pTableMeta->tableInfo.numOfTags = pChild->numOfTags; + pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; + pTableMeta->tableInfo.precision = pChild->precision; + + pTableMeta->id.tid = 0; + pTableMeta->id.uid = pChild->suid; + pTableMeta->tversion = pChild->tversion; + pTableMeta->sversion = pChild->sversion; + + memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); + + int32_t num = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < num; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + int32_t totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + return sizeof(STableMeta) + totalCols * sizeof(SSchema); +} + +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + return cMeta; +} + +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name) { + assert(pChild != NULL); + + uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* p = calloc(1, size); + + taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); + if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + pChild->sversion = p->sversion; + pChild->tversion = p->tversion; + + memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableInfo)); + int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; + + memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); + + tfree(p); + return TSDB_CODE_SUCCESS; + } else { // super table has been removed, current tableMeta is also expired. remove it here + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + + tfree(p); + return -1; + } +} + +uint32_t tscGetTableMetaMaxSize() { + return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); +} + +STableMeta* tscTableMetaClone(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + uint32_t size = tscGetTableMetaSize(pTableMeta); + STableMeta* p = calloc(1, size); + memcpy(p, pTableMeta, size); + return p; +} + + diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 5b88c9b0d02804781c221d0bc36f0e944f34b049..f1fc5ca8087ddfca0f03c84d2e93c064aad761ce 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -32,8 +32,8 @@ extern uint16_t tsSyncPort; extern uint16_t tsArbitratorPort; extern int32_t tsStatusInterval; extern int32_t tsNumOfMnodes; -extern int32_t tsEnableVnodeBak; -extern int32_t tsEnableTelemetryReporting; +extern int8_t tsEnableVnodeBak; +extern int8_t tsEnableTelemetryReporting; extern char tsEmail[]; extern char tsArbitrator[]; @@ -46,23 +46,25 @@ extern int32_t tsShellActivityTimer; extern uint32_t tsMaxTmrCtrl; extern float tsNumOfThreadsPerCore; extern int32_t tsNumOfCommitThreads; -extern float tsRatioOfQueryThreads; // todo remove it +extern float tsRatioOfQueryCores; extern int8_t tsDaylight; extern char tsTimezone[]; extern char tsLocale[]; extern char tsCharset[]; // default encode string -extern int32_t tsEnableCoreFile; +extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern char tsTempDir[]; //query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing -extern int32_t tsHalfCoresForQuery; // only 50% will be used in query processing +extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked + +extern int8_t tsKeepOriginalColumnName; // client extern int32_t tsTableMetaKeepTimer; extern int32_t tsMaxSQLStringLen; -extern int32_t tsTscEnableRecordSql; +extern int8_t tsTscEnableRecordSql; extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; @@ -91,48 +93,51 @@ extern int16_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; -extern int32_t tsUpdate; +extern int8_t tsUpdate; +extern int8_t tsCacheLastRow; // balance -extern int32_t tsEnableBalance; -extern int32_t tsAlternativeRole; +extern int8_t tsEnableBalance; +extern int8_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; extern int32_t tsMnodeEqualVnodeNum; -extern int32_t tsFlowCtrl; +extern int8_t tsEnableFlowCtrl; +extern int8_t tsEnableSlaveQuery; +extern int8_t tsEnableAdjustMaster; // restful -extern int32_t tsEnableHttpModule; +extern int8_t tsEnableHttpModule; extern int32_t tsRestRowLimit; extern uint16_t tsHttpPort; extern int32_t tsHttpCacheSessions; extern int32_t tsHttpSessionExpire; extern int32_t tsHttpMaxThreads; -extern int32_t tsHttpEnableCompress; -extern int32_t tsHttpEnableRecordSql; -extern int32_t tsTelegrafUseFieldNum; +extern int8_t tsHttpEnableCompress; +extern int8_t tsHttpEnableRecordSql; +extern int8_t tsTelegrafUseFieldNum; // mqtt -extern int32_t tsEnableMqttModule; -extern char tsMqttHostName[]; -extern char tsMqttPort[]; -extern char tsMqttUser[]; -extern char tsMqttPass[]; -extern char tsMqttClientId[]; -extern char tsMqttTopic[]; +extern int8_t tsEnableMqttModule; +extern char tsMqttHostName[]; +extern char tsMqttPort[]; +extern char tsMqttUser[]; +extern char tsMqttPass[]; +extern char tsMqttClientId[]; +extern char tsMqttTopic[]; // monitor -extern int32_t tsEnableMonitorModule; +extern int8_t tsEnableMonitorModule; extern char tsMonitorDbName[]; extern char tsInternalPass[]; extern int32_t tsMonitorInterval; // stream -extern int32_t tsEnableStream; +extern int8_t tsEnableStream; // internal -extern int32_t tsPrintAuth; -extern int32_t tscEmbedded; +extern int8_t tsPrintAuth; +extern int8_t tscEmbedded; extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; @@ -159,7 +164,7 @@ extern float tsMinimalLogDirGB; extern float tsReservedTmpDirectorySpace; extern float tsMinimalDataDirGB; extern int32_t tsTotalMemoryMB; -extern int32_t tsVersion; +extern uint32_t tsVersion; // build info extern char version[]; @@ -169,7 +174,7 @@ extern char gitinfoOfInternal[]; extern char buildinfo[]; // log -extern int32_t tsAsyncLog; +extern int8_t tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; diff --git a/src/common/inc/tulog.h b/src/common/inc/tulog.h index 2dc2895e6324053992b5cbdcad5486071b68dbc9..566da40a10e078b9789e2e1b76a8d82fe89aef46 100644 --- a/src/common/inc/tulog.h +++ b/src/common/inc/tulog.h @@ -23,7 +23,7 @@ extern "C" { #include "tlog.h" extern int32_t uDebugFlag; -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 17eb5714e0272bcebe21de84d938f94708b7c4d9..8fa17f87511ca02759b5700dd59735a526da7fe9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -25,7 +25,6 @@ #include "tutil.h" #include "tlocale.h" #include "ttimezone.h" -#include "tsync.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -40,8 +39,8 @@ uint16_t tsSyncPort = 6040; uint16_t tsArbitratorPort = 6042; int32_t tsStatusInterval = 1; // second int32_t tsNumOfMnodes = 3; -int32_t tsEnableVnodeBak = 1; -int32_t tsEnableTelemetryReporting = 1; +int8_t tsEnableVnodeBak = 1; +int8_t tsEnableTelemetryReporting = 1; char tsEmail[TSDB_FQDN_LEN] = {0}; // common @@ -52,12 +51,12 @@ int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 1; -float tsRatioOfQueryThreads = 0.5f; +float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string -int32_t tsEnableCoreFile = 0; +int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; @@ -74,7 +73,7 @@ int32_t tsCompressMsgSize = -1; // client int32_t tsTableMetaKeepTimer = 7200; // second int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; -int32_t tsTscEnableRecordSql = 0; +int8_t tsTscEnableRecordSql = 0; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -107,8 +106,11 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance // positive value (in MB) int32_t tsQueryBufferSize = -1; -// only 50% cpu will be used in query processing in dnode -int32_t tsHalfCoresForQuery = 0; +// in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. +int32_t tsRetrieveBlockingModel = 0; + +// last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name +int8_t tsKeepOriginalColumnName = 0; // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; @@ -124,33 +126,36 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; +int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; +int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; // balance -int32_t tsEnableBalance = 1; -int32_t tsAlternativeRole = 0; -int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400*100; // seconds 10days +int8_t tsEnableBalance = 1; +int8_t tsAlternativeRole = 0; +int32_t tsBalanceInterval = 300; // seconds +int32_t tsOfflineThreshold = 86400 * 100; // seconds 10days int32_t tsMnodeEqualVnodeNum = 4; -int32_t tsFlowCtrl = 1; +int8_t tsEnableFlowCtrl = 1; +int8_t tsEnableSlaveQuery = 1; +int8_t tsEnableAdjustMaster = 1; // restful -int32_t tsEnableHttpModule = 1; +int8_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] int32_t tsHttpCacheSessions = 1000; int32_t tsHttpSessionExpire = 36000; int32_t tsHttpMaxThreads = 2; -int32_t tsHttpEnableCompress = 1; -int32_t tsHttpEnableRecordSql = 0; -int32_t tsTelegrafUseFieldNum = 0; +int8_t tsHttpEnableCompress = 1; +int8_t tsHttpEnableRecordSql = 0; +int8_t tsTelegrafUseFieldNum = 0; // mqtt -int32_t tsEnableMqttModule = 0; // not finished yet, not started it by default +int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org"; char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883"; char tsMqttUser[TSDB_MQTT_USER_LEN] = {0}; @@ -159,24 +164,24 @@ char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber"; char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // # // monitor -int32_t tsEnableMonitorModule = 1; +int8_t tsEnableMonitorModule = 1; char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsInternalPass[] = "secretkey"; int32_t tsMonitorInterval = 30; // seconds // stream -int32_t tsEnableStream = 1; +int8_t tsEnableStream = 1; // internal -int32_t tsPrintAuth = 0; -int32_t tscEmbedded = 0; -char configDir[TSDB_FILENAME_LEN] = {0}; -char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsDataDir[TSDB_FILENAME_LEN] = {0}; -char tsScriptDir[TSDB_FILENAME_LEN] = {0}; -char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; +int8_t tsPrintAuth = 0; +int8_t tscEmbedded = 0; +char configDir[TSDB_FILENAME_LEN] = {0}; +char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsDataDir[TSDB_FILENAME_LEN] = {0}; +char tsScriptDir[TSDB_FILENAME_LEN] = {0}; +char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; /* * minimum scale for whole system, millisecond by default @@ -196,10 +201,10 @@ float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; float tsAvailTmpDirectorySpace = 0; float tsAvailDataDirGB = 0; -float tsReservedTmpDirectorySpace = 0.1f; -float tsMinimalDataDirGB = 0.5f; +float tsReservedTmpDirectorySpace = 1.0f; +float tsMinimalDataDirGB = 1.0f; int32_t tsTotalMemoryMB = 0; -int32_t tsVersion = 0; +uint32_t tsVersion = 0; // log int32_t tsNumOfLogLines = 10000000; @@ -273,12 +278,16 @@ bool taosCfgDynamicOptions(char *msg) { for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (cfg->valType != TAOS_CFG_VTYPE_INT32) continue; + if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; - *((int32_t *)cfg->ptr) = vint; + if (cfg->valType != TAOS_CFG_VTYPE_INT32) { + *((int32_t *)cfg->ptr) = vint; + } else { + *((int8_t *)cfg->ptr) = (int8_t)vint; + } if (strncasecmp(cfg->option, "monitor", olen) == 0) { if (1 == vint) { @@ -444,12 +453,12 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "ratioOfQueryThreads"; - cfg.ptr = &tsRatioOfQueryThreads; + cfg.option = "ratioOfQueryCores"; + cfg.ptr = &tsRatioOfQueryCores; cfg.valType = TAOS_CFG_VTYPE_FLOAT; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0.1f; - cfg.maxValue = 0.9f; + cfg.minValue = 0.0f; + cfg.maxValue = 2.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -466,7 +475,7 @@ static void doInitGlobalConfig(void) { cfg.option = "vnodeBak"; cfg.ptr = &tsEnableVnodeBak; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -476,7 +485,7 @@ static void doInitGlobalConfig(void) { cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -486,7 +495,7 @@ static void doInitGlobalConfig(void) { cfg.option = "balance"; cfg.ptr = &tsEnableBalance; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -507,7 +516,7 @@ static void doInitGlobalConfig(void) { // 0-any; 1-mnode; 2-vnode cfg.option = "role"; cfg.ptr = &tsAlternativeRole; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 2; @@ -540,7 +549,7 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsOfflineThreshold; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 5; + cfg.minValue = 3; cfg.maxValue = 7200000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; @@ -809,7 +818,7 @@ static void doInitGlobalConfig(void) { cfg.option = "update"; cfg.ptr = &tsUpdate; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_DB_UPDATE; cfg.maxValue = TSDB_MAX_DB_UPDATE; @@ -887,8 +896,8 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); - cfg.option = "halfCoresForQuery"; - cfg.ptr = &tsHalfCoresForQuery; + cfg.option = "retrieveBlockingModel"; + cfg.ptr = &tsRetrieveBlockingModel; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; @@ -897,6 +906,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "keepColumnName"; + cfg.ptr = &tsKeepOriginalColumnName; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + // locale & charset cfg.option = "timezone"; cfg.ptr = tsTimezone; @@ -992,8 +1011,28 @@ static void doInitGlobalConfig(void) { // module configs cfg.option = "flowctrl"; - cfg.ptr = &tsFlowCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.ptr = &tsEnableFlowCtrl; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "slaveQuery"; + cfg.ptr = &tsEnableSlaveQuery; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "adjustMaster"; + cfg.ptr = &tsEnableAdjustMaster; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1003,7 +1042,7 @@ static void doInitGlobalConfig(void) { cfg.option = "http"; cfg.ptr = &tsEnableHttpModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1013,7 +1052,7 @@ static void doInitGlobalConfig(void) { cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1023,7 +1062,7 @@ static void doInitGlobalConfig(void) { cfg.option = "monitor"; cfg.ptr = &tsEnableMonitorModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1033,7 +1072,7 @@ static void doInitGlobalConfig(void) { cfg.option = "stream"; cfg.ptr = &tsEnableStream; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1043,7 +1082,7 @@ static void doInitGlobalConfig(void) { cfg.option = "httpEnableRecordSql"; cfg.ptr = &tsHttpEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1053,7 +1092,7 @@ static void doInitGlobalConfig(void) { cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1104,7 +1143,7 @@ static void doInitGlobalConfig(void) { cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 1; @@ -1305,7 +1344,7 @@ static void doInitGlobalConfig(void) { cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1315,7 +1354,7 @@ static void doInitGlobalConfig(void) { cfg.option = "enableCoreFile"; cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1431,17 +1470,28 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } + if (tsMaxTablePerVnode < tsMinTablePerVnode) { + uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", + tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode); + tsMaxTablePerVnode = tsMinTablePerVnode; + } + // todo refactor tsVersion = 0; - for (int i = 0; i < 10; i++) { + for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { if (version[i] >= '0' && version[i] <= '9') { - tsVersion = tsVersion * 10 + (version[i] - '0'); + ver = ver * 10 + (version[i] - '0'); + } else if (version[i] == '.') { + tsVersion |= ver & 0xFF; + tsVersion <<= 8; + + ver = 0; } else if (version[i] == 0) { + tsVersion |= ver & 0xFF; + break; } } - - tsVersion = 10 * tsVersion; tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035] tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE; // udp/tcp diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index fc00f50a7af0cd6b92262438baa293dda7066a07..571ec2e0ddd4f141081debb39f79fbe710266c42 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -705,7 +705,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL; return 0; } else { - double value; + double value = -1; int32_t ret; ret = convertToDouble(pVariant->pz, pVariant->nLen, &value); if ((errno == ERANGE && (float)value == -1) || (ret != 0)) { diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index ec77d9049a719dabfd1a7c1122a209e201861944..32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944 +Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 656741ea16fd607212711fb694e3e20e6007b2ac..995babdb2b7827267d6a4b68cd9d380671823543 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -144,18 +144,9 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; - let currOffset = 0; - // every 4 bytes, a character is encoded; - while (currOffset < data.length) { - let dataEntry = data.slice(currOffset, currOffset + nbytes); //one entry in a row under a column; - if (dataEntry.readInt64LE(0) == FieldTypes.C_NCHAR_NULL) { - res.push(null); - } - else { - res.push(dataEntry.toString("utf16le").replace(/\u0000/g, "")); - } - currOffset += nbytes; - } + let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; + //TODO: should use the correct character encoding + res.push(dataEntry.toString("utf-8")); return res; } @@ -349,11 +340,13 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let num_of_rows = this.libtaos.taos_fetch_block(result, pblock) - if (num_of_rows == 0) { + //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data + let pblock = this.libtaos.taos_fetch_row(result); + let num_of_rows = 1; + if (ref.isNull(pblock) == true) { return {block:null, num_of_rows:0}; } + var fieldL = this.libtaos.taos_fetch_lengths(result); let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); @@ -361,26 +354,28 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { var fieldlens = []; if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + let plen = ref.reinterpret(fieldL, 4, i*4); let len = plen.readInt32LE(0); - fieldlens.push(len); + fieldlens.push(len); } } let blocks = new Array(fields.length); blocks.fill(null); - num_of_rows = Math.abs(num_of_rows); + //num_of_rows = Math.abs(num_of_rows); let offset = 0; - pblock = pblock.deref(); for (let i = 0; i < fields.length; i++) { pdata = ref.reinterpret(pblock,8,i*8); - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + } } return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} } @@ -446,14 +441,18 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, } if (numOfRows2 > 0){ for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row,8,i*8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); diff --git a/src/connector/nodejs/nodetaos/taosresult.js b/src/connector/nodejs/nodetaos/taosresult.js index fd82f4e236508eff46c54f6f156b592a43319c20..4138ebbec6e1b792691d17a25b7c18d35b6a922a 100644 --- a/src/connector/nodejs/nodetaos/taosresult.js +++ b/src/connector/nodejs/nodetaos/taosresult.js @@ -25,6 +25,7 @@ function TaosResult(data, fields) { * @function pretty * @since 1.0.6 */ + TaosResult.prototype.pretty = function pretty() { let fieldsStr = ""; let sizing = []; @@ -46,8 +47,7 @@ TaosResult.prototype.pretty = function pretty() { row.data.forEach((entry, i) => { if (this.fields[i]._field.type == 9) { entry = entry.toTaosString(); - } - else { + } else { entry = entry == null ? 'null' : entry.toString(); } rowStr += entry diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index 3f0600a09ca73c3fadbed1b7e65c51b30004edf5..2d5cf45e1db9a65ce86d998afea6cd689bf63317 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.1", + "version": "2.0.4", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "scripts": { diff --git a/src/connector/nodejs/test/test.js b/src/connector/nodejs/test/test.js index 73dac8b26ce07ff159ce3de535776e2495d1ebde..bf4bb2c54188d3eb0f9c7fb5306912effc7b0760 100644 --- a/src/connector/nodejs/test/test.js +++ b/src/connector/nodejs/test/test.js @@ -48,6 +48,7 @@ for (let i = 0; i < 10000; i++) { // Select console.log('select * from td_connector_test.all_types limit 3 offset 100;'); c1.execute('select * from td_connector_test.all_types limit 2 offset 100;'); + var d = c1.fetchall(); console.log(c1.fields); console.log(d); @@ -77,13 +78,33 @@ c1.query('select stddev(_double), stddev(_bigint), stddev(_float) from all_types }) // Binding arguments, and then using promise -var q = c1.query('select * from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) +var q = c1.query('select _nchar from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) console.log(q.query); q.execute().then(function(r) { r.pretty(); }); +// test query null value +c1.execute("create table if not exists td_connector_test.weather(ts timestamp, temperature float, humidity int) tags(location nchar(64))"); +c1.execute("insert into t1 using weather tags('北京') values(now, 11.11, 11)"); +c1.execute("insert into t1(ts, temperature) values(now, 22.22)"); +c1.execute("insert into t1(ts, humidity) values(now, 33)"); +c1.query('select * from test.t1', true).then(function (result) { + result.pretty(); +}); + +var q = c1.query('select * from td_connector_test.weather'); +console.log(q.query); +q.execute().then(function(r) { + r.pretty(); +}); + +function sleep(sleepTime) { + for(var start = +new Date; +new Date - start <= sleepTime; ) { } +} + +sleep(10000); // Raw Async Testing (Callbacks, not promises) function cb2(param, result, rowCount, rd) { @@ -129,16 +150,21 @@ setTimeout(function(){ c1.fetchall_a(thisRes, cb4, param); },100); + // Async through promises var aq = c1.query('select count(*) from td_connector_test.all_types;',false); aq.execute_a().then(function(data) { data.pretty(); }); -c1.query('describe td_connector_test.stabletest;').execute_a().then(r=> r.pretty()); + +c1.query('describe td_connector_test.stabletest').execute_a().then(function(r){ + r.pretty() +}); + setTimeout(function(){ c1.query('drop database td_connector_test;'); },200); + setTimeout(function(){ conn.close(); },2000); - diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index e278c3a7ccd145585b70d3488abaf1a9dceab571..de76c30e8e1232947adee19d22744570e4a1eb43 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -97,7 +97,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { pthread_mutex_init(&pContext->mutex, NULL); - cInfo("vgId:%d, CQ is opened", pContext->vgId); + cDebug("vgId:%d, CQ is opened", pContext->vgId); return pContext; } @@ -131,7 +131,7 @@ void cqClose(void *handle) { taosTmrCleanUp(pContext->tmrCtrl); pContext->tmrCtrl = NULL; - cInfo("vgId:%d, CQ is closed", pContext->vgId); + cDebug("vgId:%d, CQ is closed", pContext->vgId); free(pContext); } @@ -142,7 +142,7 @@ void cqStart(void *handle) { SCqContext *pContext = handle; if (pContext->dbConn || pContext->master) return; - cInfo("vgId:%d, start all CQs", pContext->vgId); + cDebug("vgId:%d, start all CQs", pContext->vgId); pthread_mutex_lock(&pContext->mutex); pContext->master = 1; @@ -298,7 +298,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { if (pObj->pStream) { tscSetStreamDestTable(pObj->pStream, pObj->dstTable); pContext->num++; - cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr); + cDebug("vgId:%d, id:%d CQ:%s is opened", pContext->vgId, pObj->tid, pObj->sqlStr); } else { cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr); } diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h index d74303f3252e1fa54e174876bab5758be541955e..896b3f574c2e0d02a0d62048a411fa484d16130b 100644 --- a/src/dnode/inc/dnodeCfg.h +++ b/src/dnode/inc/dnodeCfg.h @@ -25,6 +25,7 @@ int32_t dnodeInitCfg(); void dnodeCleanupCfg(); void dnodeUpdateCfg(SDnodeCfg *cfg); int32_t dnodeGetDnodeId(); +void dnodeGetClusterId(char *clusterId); void dnodeGetCfg(int32_t *dnodeId, char *clusterId); #ifdef __cplusplus diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c index 89249d773b7c67b8939a7a70f65882f89f8f5190..f495dbe285c196f3e32d8a0ace8736c8f44d668d 100644 --- a/src/dnode/src/dnodeCfg.c +++ b/src/dnode/src/dnodeCfg.c @@ -51,6 +51,12 @@ int32_t dnodeGetDnodeId() { return dnodeId; } +void dnodeGetClusterId(char *clusterId) { + pthread_mutex_lock(&tsCfgMutex); + tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&tsCfgMutex); +} + void dnodeGetCfg(int32_t *dnodeId, char *clusterId) { pthread_mutex_lock(&tsCfgMutex); *dnodeId = tsCfg.dnodeId; diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index 7c385a889d10fcedd819f6eb10cbdaaf4910739f..dc89487f8b4c5ef2d2434e815d2ba1d8a0311515 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -311,6 +311,14 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); + if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) { + if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) || + (epSet.port[i] == tsServerPort && forShell)) { + epSet.inUse = (i + 1) % epSet.numOfEps; + dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse); + } + } + epSet.port[i] = htons(epSet.port[i]); } diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 0fc6400d99f8dbcbc57caa2ed1265e36e18f1a93..9027c346f57c20d91d943c0df8d061c931240101 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -16,9 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tqueue.h" -#include "twal.h" #include "mnode.h" -#include "dnodeVMgmt.h" #include "dnodeMInfos.h" #include "dnodeMRead.h" diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 414b66653d123b785643cfdc96b429edbe8d58ad..4bd1eae25ba37a18c03cb0fcb6989105b0799c5e 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -18,7 +18,6 @@ #include "ttimer.h" #include "tqueue.h" #include "mnode.h" -#include "dnodeVMgmt.h" #include "dnodeMInfos.h" #include "dnodeMWrite.h" @@ -185,7 +184,19 @@ void dnodeReprocessMWriteMsg(void *pMsg) { dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); - dnodeSendRedirectMsg(pMsg, true); + if (pWrite->pBatchMasterMsg) { + ++pWrite->pBatchMasterMsg->received; + if (pWrite->pBatchMasterMsg->successed + pWrite->pBatchMasterMsg->received + >= pWrite->pBatchMasterMsg->expected) { + dnodeSendRedirectMsg(&pWrite->rpcMsg, true); + dnodeFreeMWriteMsg(pWrite); + } + + mnodeDestroySubMsg(pWrite); + + return; + } + dnodeSendRedirectMsg(&pWrite->rpcMsg, true); dnodeFreeMWriteMsg(pWrite); } else { dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 730dcf3681b4c9c6c14056069c90f4dfe2006666..517a9e9bc83fb454bb4e2c43e31cbd8c90ad91cd 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -113,6 +113,7 @@ static void dnodeCleanupTmr() { int32_t dnodeInitSystem() { dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); tscEmbedded = 1; + taosIgnSIGPIPE(); taosBlockSIGPIPE(); taosResolveCRC(); taosInitGlobalCfg(); @@ -120,7 +121,6 @@ int32_t dnodeInitSystem() { taosSetCoreDump(); taosInitNotes(); dnodeInitTmr(); - signal(SIGPIPE, SIG_IGN); if (dnodeCreateDir(tsLogDir) < 0) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); @@ -147,6 +147,7 @@ int32_t dnodeInitSystem() { dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); + dnodeReportStep("TDengine", "initialized successfully", 1); dInfo("TDengine is initialized successfully"); return 0; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index d76af4e3dcf4027191e7cb92e0a9701e2c30c942..79cc70005b6e83bc1c455abb7cd3709ae29a8643 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -70,8 +70,7 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep; - int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore; - numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0); + int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0; if (numOfThreads < 1) { numOfThreads = 1; } diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c index 0f535b9470ad53a62b294e82e118dd1e5e78175d..2354b1d5a382d4ca7dea523cfe675883ae9d1570 100644 --- a/src/dnode/src/dnodeStep.c +++ b/src/dnode/src/dnodeStep.c @@ -57,12 +57,13 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) { int32_t code = (*pStep->initFp)(); if (code != 0) { - dDebug("step:%s will init", pStep->name); + dDebug("step:%s will cleanup", pStep->name); taosStepCleanupImp(pSteps, step); return code; } + dInfo("step:%s is initialized", pStep->name); - dnodeReportStep(pStep->name, "Initialization complete", step + 1 >= stepSize); + dnodeReportStep(pStep->name, "Initialization complete", 0); } return 0; diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 85f0137d896ff65440117dcc6fc6e2a3c1a28038..ff9598ecc563ac48bac051027385ab188e94ce75 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -195,7 +195,7 @@ static void addRuntimeInfo(SBufferWriter* bw) { static void sendTelemetryReport() { char buf[128]; - uint32_t ip = taosGetIpFromFqdn(TELEMETRY_SERVER); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; @@ -308,4 +308,4 @@ void dnodeCleanupTelemetry() { pthread_join(tsTelemetryThread, NULL); tsem_destroy(&tsExitSem); } -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c index e3cf0820ae2073b7e380e52606baf4e3f6e66d3f..bc24d1bf623ec014dd4a4ad35442218549aaf335 100644 --- a/src/dnode/src/dnodeVMgmt.c +++ b/src/dnode/src/dnodeVMgmt.c @@ -129,7 +129,8 @@ static void *dnodeProcessMgmtQueue(void *wparam) { static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { SCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->cfg.vgId = htonl(pCreate->cfg.vgId); - pCreate->cfg.cfgVersion = htonl(pCreate->cfg.cfgVersion); + pCreate->cfg.dbCfgVersion = htonl(pCreate->cfg.dbCfgVersion); + pCreate->cfg.vgCfgVersion = htonl(pCreate->cfg.vgCfgVersion); pCreate->cfg.maxTables = htonl(pCreate->cfg.maxTables); pCreate->cfg.cacheBlockSize = htonl(pCreate->cfg.cacheBlockSize); pCreate->cfg.totalBlocks = htonl(pCreate->cfg.totalBlocks); @@ -142,7 +143,7 @@ static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod); pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); - for (int32_t j = 0; j < pCreate->cfg.replications; ++j) { + for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) { pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); } @@ -216,4 +217,4 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { dnodeStartMnode(&pCfg->mnodes); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 0d4add2a5c4c793667bb36300fbe3baf1894ce6a..ea738661ce2813e13468ad91b4dc1d54775db21f 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -26,16 +26,20 @@ static SWorkerPool tsVQueryWP; static SWorkerPool tsVFetchWP; int32_t dnodeInitVRead() { + const int32_t maxFetchThreads = 4; + + // calculate the available query thread + float threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1); + tsVQueryWP.name = "vquery"; tsVQueryWP.workerFp = dnodeProcessReadQueue; - tsVQueryWP.min = tsNumOfCores; - tsVQueryWP.max = tsNumOfCores/* * tsNumOfThreadsPerCore*/; -// if (tsVQueryWP.max <= tsVQueryWP.min * 2) tsVQueryWP.max = 2 * tsVQueryWP.min; + tsVQueryWP.min = (int32_t) threadsForQuery; + tsVQueryWP.max = tsVQueryWP.min; if (tWorkerInit(&tsVQueryWP) != 0) return -1; tsVFetchWP.name = "vfetch"; tsVFetchWP.workerFp = dnodeProcessReadQueue; - tsVFetchWP.min = MIN(4, tsNumOfCores); + tsVFetchWP.min = MIN(maxFetchThreads, tsNumOfCores); tsVFetchWP.max = tsVFetchWP.min; if (tWorkerInit(&tsVFetchWP) != 0) return -1; @@ -50,6 +54,7 @@ void dnodeCleanupVRead() { void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { int32_t queuedMsgNum = 0; int32_t leftLen = pMsg->contLen; + int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID; char * pCont = pMsg->pCont; while (leftLen > 0) { @@ -60,7 +65,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { assert(pHead->contLen > 0); void *pVnode = vnodeAcquire(pHead->vgId); if (pVnode != NULL) { - int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; vnodeRelease(pVnode); } @@ -70,7 +75,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { } if (queuedMsgNum == 0) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code}; rpcSendResponse(&rpcRsp); } diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index a5ae8ac83063c599ad4c215bd0a7fa4468810580..775ced09900612da76fff5a60d42839b83fcb133 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -188,6 +188,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { int32_t numOfMsgs; int32_t qtype; + taosBlockSIGPIPE(); dDebug("dnode vwrite worker:%d is running", pWorker->workerId); while (1) { diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index 85b997d94c885c131aa033a17b486262f17fc334..c62d5a8207c8cd5f540e3ff861d44cd55c7c8220 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -33,6 +33,8 @@ typedef struct { extern void * tsDnodeTmr; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime = 0; +static int32_t tsOpenVnodes = 0; +static int32_t tsTotalVnodes = 0; static void dnodeSendStatusMsg(void *handle, void *tmrId); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); @@ -71,7 +73,8 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { if (*numOfVnodes >= TSDB_MAX_VNODES) { dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES); - continue; + closedir(dir); + return TSDB_CODE_DND_TOO_MANY_VNODES; } else { vnodeList[*numOfVnodes - 1] = vnode; } @@ -84,21 +87,27 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { static void *dnodeOpenVnode(void *param) { SOpenVnodeThread *pThread = param; + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); for (int32_t v = 0; v < pThread->vnodeNum; ++v) { int32_t vgId = pThread->vnodeList[v]; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", vgId, tsOpenVnodes, tsTotalVnodes); + dnodeReportStep("open-vnodes", stepDesc, 0); + if (vnodeOpen(vgId) < 0) { dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex); pThread->failed++; } else { - dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex); + dDebug("vgId:%d, is opened by thread:%d", vgId, pThread->threadIndex); pThread->opened++; } + + atomic_add_fetch_32(&tsOpenVnodes, 1); } - dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, pThread->failed); return NULL; } @@ -107,6 +116,7 @@ int32_t dnodeInitVnodes() { int32_t vnodeList[TSDB_MAX_VNODES] = {0}; int32_t numOfVnodes = 0; int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes); + tsTotalVnodes = numOfVnodes; if (status != TSDB_CODE_SUCCESS) { dInfo("get dnode list failed"); @@ -127,7 +137,7 @@ int32_t dnodeInitVnodes() { pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v]; } - dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes); + dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); for (int32_t t = 0; t < threadNum; ++t) { SOpenVnodeThread *pThread = &threads[t]; @@ -156,7 +166,7 @@ int32_t dnodeInitVnodes() { } free(threads); - dInfo("there are total vnodes:%d, openned:%d", numOfVnodes, openVnodes); + dInfo("there are total vnodes:%d, opened:%d", numOfVnodes, openVnodes); if (failedVnodes != 0) { dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes); @@ -236,12 +246,11 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { pStatus->lastReboot = htonl(tsRebootTime); pStatus->numOfCores = htons((uint16_t) tsNumOfCores); pStatus->diskAvailable = tsAvailDataDirGB; - pStatus->alternativeRole = (uint8_t) tsAlternativeRole; + pStatus->alternativeRole = tsAlternativeRole; tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); // fill cluster cfg parameters pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes); - pStatus->clusterCfg.enableBalance = htonl(tsEnableBalance); pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum); pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold); pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); @@ -253,7 +262,12 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); + + pStatus->clusterCfg.enableBalance = tsEnableBalance; + pStatus->clusterCfg.flowCtrl = tsEnableFlowCtrl; + pStatus->clusterCfg.slaveQuery = tsEnableSlaveQuery; + pStatus->clusterCfg.adjustMaster = tsEnableAdjustMaster; vnodeBuildStatusMsg(pStatus); contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); @@ -275,4 +289,4 @@ void dnodeSendStatusMsgToMnode() { dInfo("force send status msg to mnode"); taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer); } -} \ No newline at end of file +} diff --git a/src/inc/dnode.h b/src/inc/dnode.h index dd41360e68e1868dc1b947cfc5ab4188e96fc320..877738778b022c9c7d38a3801beb5cdc86ff9f4d 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -36,6 +36,8 @@ bool dnodeIsMasterEp(char *ep); void dnodeGetEpSetForPeer(SRpcEpSet *epSet); void dnodeGetEpSetForShell(SRpcEpSet *epSet); int32_t dnodeGetDnodeId(); +void dnodeGetClusterId(char *clusterId); + void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); bool dnodeStartMnode(SMInfos *pMinfos); @@ -80,4 +82,4 @@ void dnodeReportStep(char *name, char *desc, int8_t finished); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/inc/mnode.h b/src/inc/mnode.h index bdc30b0c46ced0961715bd48623fdb9e52fb440e..800d767eedac3d4680f74564db00e7645917d13b 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -42,11 +42,12 @@ typedef struct SMnodeMsg { struct SVgObj * pVgroup; struct STableObj *pTable; struct SSTableObj*pSTable; + struct SMnodeMsg *pBatchMasterMsg; SMnodeRsp rpcRsp; - int8_t received; - int8_t successed; - int8_t expected; - int8_t retry; + int16_t received; + int16_t successed; + int16_t expected; + int16_t retry; int32_t incomingTs; int32_t code; void * pObj; @@ -57,6 +58,7 @@ typedef struct SMnodeMsg { void * mnodeCreateMsg(SRpcMsg *pRpcMsg); int32_t mnodeInitMsg(SMnodeMsg *pMsg); void mnodeCleanupMsg(SMnodeMsg *pMsg); +void mnodeDestroySubMsg(SMnodeMsg *pSubMsg); int32_t mnodeInitSystem(); int32_t mnodeStartSystem(); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 0cc06be1dbcafbaba1ab00b041aff7af9db6e445..7f1ed408154e6a99232afce3eb927b64f0412c5c 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -369,6 +369,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_DB_UPDATE 1 #define TSDB_DEFAULT_DB_UPDATE_OPTION 0 +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 1 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 + #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second @@ -432,7 +436,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_HTTP 11 #define TSDB_PORT_ARBITRATOR 12 -#define TSDB_MAX_WAL_SIZE (1024*1024) +#define TSDB_MAX_WAL_SIZE (1024*1024*2) typedef enum { TAOS_QTYPE_RPC = 0, diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index be33262f7f75f20485c977c3b3a7378a6021481a..641b65749937438f14744355136b9b6f0aecef3f 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -67,6 +67,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid re TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Client and server's time is not synchronized") TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION, 0, 0x0016, "Invalid app version") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") @@ -193,6 +194,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, 0, 0x0404, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, 0, 0x0405, "Too many vnode directories") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress") @@ -208,9 +210,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Database memory is full for commit failed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, 0, 0x050C, "Database memory is full for waiting commit") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_DROPPING, 0, 0x050D, "Database is dropping") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_BALANCING, 0, 0x050E, "Database is balancing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Database write operation denied") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, 0, 0x0513, "Database is syncing") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "Invalid table ID") @@ -266,6 +270,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sy TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TOO_MANY_FWDINFO, 0, 0x0904, "Too many sync fwd infos") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_PROTOCOL, 0, 0x0905, "Mismatched protocol") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_CLUSTERID, 0, 0x0906, "Mismatched clusterId") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_SIGNATURE, 0, 0x0907, "Mismatched signature") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CHECKSUM, 0, 0x0908, "Invalid msg checksum") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGLEN, 0, 0x0909, "Invalid msg length") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, 0, 0x090A, "Invalid msg type") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 27d857ce1401ff74ecff72e92354f1ee19d568eb..2dee6dc3bb20c7397ef435d4b7f37e5861f8891e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -266,6 +266,7 @@ typedef struct { } SMDCreateTableMsg; typedef struct { + int32_t len; // one create table message char tableId[TSDB_TABLE_FNAME_LEN]; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int8_t igExists; @@ -273,9 +274,13 @@ typedef struct { int16_t numOfTags; int16_t numOfColumns; int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string - int32_t contLen; int8_t reserved[16]; char schema[]; +} SCreateTableMsg; + +typedef struct { + int32_t numOfTables; + int32_t contLen; } SCMCreateTableMsg; typedef struct { @@ -319,6 +324,7 @@ typedef struct { typedef struct { char acctId[TSDB_ACCT_LEN]; char serverVersion[TSDB_VERSION_LEN]; + char clusterId[TSDB_CLUSTER_ID_LEN]; int8_t writeAuth; int8_t superAuth; int8_t reserved1; @@ -470,6 +476,7 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; uint16_t tagCondLen; // tag length in current query + uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx @@ -488,6 +495,7 @@ typedef struct { int32_t tsNumOfBlocks; // ts comp block numbers int32_t tsOrder; // ts comp block order int32_t numOfTags; // number of tags columns involved + int32_t sqlstrLen; // sql query string SColumnInfo colList[]; } SQueryTableMsg; @@ -512,15 +520,17 @@ typedef struct SRetrieveTableRsp { } SRetrieveTableRsp; typedef struct { - int32_t vgId; - int32_t cfgVersion; - int64_t totalStorage; - int64_t compStorage; - int64_t pointsWritten; - uint8_t status; - uint8_t role; - uint8_t replica; - uint8_t reserved[5]; + int32_t vgId; + int32_t dbCfgVersion; + int64_t totalStorage; + int64_t compStorage; + int64_t pointsWritten; + uint64_t vnodeVersion; + int32_t vgCfgVersion; + uint8_t status; + uint8_t role; + uint8_t replica; + uint8_t reserved; } SVnodeLoad; typedef struct { @@ -543,7 +553,8 @@ typedef struct { int8_t quorum; int8_t ignoreExist; int8_t update; - int8_t reserve[9]; + int8_t cacheLastRow; + int8_t reserve[8]; } SCreateDbMsg, SAlterDbMsg; typedef struct { @@ -598,7 +609,6 @@ typedef struct { typedef struct { int32_t numOfMnodes; // tsNumOfMnodes - int32_t enableBalance; // tsEnableBalance int32_t mnodeEqualVnodeNum; // tsMnodeEqualVnodeNum int32_t offlineThreshold; // tsOfflineThreshold int32_t statusInterval; // tsStatusInterval @@ -609,6 +619,11 @@ typedef struct { int64_t checkTime; // 1970-01-01 00:00:00.000 char locale[TSDB_LOCALE_LEN]; // tsLocale char charset[TSDB_LOCALE_LEN]; // tsCharset + int8_t enableBalance; // tsEnableBalance + int8_t flowCtrl; + int8_t slaveQuery; + int8_t adjustMaster; + int8_t reserved[4]; } SClusterCfg; typedef struct { @@ -636,7 +651,7 @@ typedef struct { typedef struct { uint32_t vgId; - int32_t cfgVersion; + int32_t dbCfgVersion; int32_t maxTables; int32_t cacheBlockSize; int32_t totalBlocks; @@ -651,11 +666,14 @@ typedef struct { int8_t precision; int8_t compression; int8_t walLevel; - int8_t replications; + int8_t vgReplica; int8_t wals; int8_t quorum; int8_t update; - int8_t reserved[15]; + int8_t cacheLastRow; + int32_t vgCfgVersion; + int8_t dbReplica; + int8_t reserved[9]; } SVnodeCfg; typedef struct { @@ -709,7 +727,6 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; char tableId[TSDB_TABLE_FNAME_LEN]; // table id - char sTableId[TSDB_TABLE_FNAME_LEN]; uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -719,6 +736,9 @@ typedef struct STableMetaMsg { int32_t tid; uint64_t uid; SVgroupMsg vgroup; + + char sTableName[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; SSchema schema[]; } STableMetaMsg; @@ -730,8 +750,8 @@ typedef struct SMultiTableMeta { typedef struct { int32_t dataLen; - char name[TSDB_TABLE_FNAME_LEN]; - char data[TSDB_MAX_TAGS_LEN + TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * TSDB_MAX_TAGS]; + char name[TSDB_TABLE_FNAME_LEN]; + char *data; } STagData; /* diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 04d6c7881556defb93b0b65cefff6f69b9bf7cbc..262bf30309ae8bc8bb24b844a88acbfd2290d0c0 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -66,6 +66,7 @@ typedef struct { int8_t precision; int8_t compression; int8_t update; + int8_t cacheLastRow; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS @@ -119,7 +120,7 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg); int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); -TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); +// TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 1303195ef149a98b6ddd7ffe55615d27eaeba04d..4dae86bbed538a0801251f62723a471215655e24 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -119,11 +119,6 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *); extern char *syncRole[]; //global configurable parameters -extern int32_t tsMaxSyncNum; -extern int32_t tsSyncTcpThreads; -extern int32_t tsMaxWatchFiles; -extern int32_t tsSyncTimer; -extern int32_t tsMaxFwdInfo; extern int32_t sDebugFlag; extern char tsArbitrator[]; extern uint16_t tsSyncPort; diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 0a5a3d2fa402c3dd03f1feef8cf05e446922972b..7bd7b228cbbdbfa4d5d6db2d6fe9c0bb647c9fea 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -114,114 +114,115 @@ #define TK_COMP 96 #define TK_PRECISION 97 #define TK_UPDATE 98 -#define TK_LP 99 -#define TK_RP 100 -#define TK_TAGS 101 -#define TK_USING 102 -#define TK_AS 103 -#define TK_COMMA 104 -#define TK_NULL 105 -#define TK_SELECT 106 -#define TK_UNION 107 -#define TK_ALL 108 -#define TK_FROM 109 -#define TK_VARIABLE 110 -#define TK_INTERVAL 111 -#define TK_FILL 112 -#define TK_SLIDING 113 -#define TK_ORDER 114 -#define TK_BY 115 -#define TK_ASC 116 -#define TK_DESC 117 -#define TK_GROUP 118 -#define TK_HAVING 119 -#define TK_LIMIT 120 -#define TK_OFFSET 121 -#define TK_SLIMIT 122 -#define TK_SOFFSET 123 -#define TK_WHERE 124 -#define TK_NOW 125 -#define TK_RESET 126 -#define TK_QUERY 127 -#define TK_ADD 128 -#define TK_COLUMN 129 -#define TK_TAG 130 -#define TK_CHANGE 131 -#define TK_SET 132 -#define TK_KILL 133 -#define TK_CONNECTION 134 -#define TK_STREAM 135 -#define TK_COLON 136 -#define TK_ABORT 137 -#define TK_AFTER 138 -#define TK_ATTACH 139 -#define TK_BEFORE 140 -#define TK_BEGIN 141 -#define TK_CASCADE 142 -#define TK_CLUSTER 143 -#define TK_CONFLICT 144 -#define TK_COPY 145 -#define TK_DEFERRED 146 -#define TK_DELIMITERS 147 -#define TK_DETACH 148 -#define TK_EACH 149 -#define TK_END 150 -#define TK_EXPLAIN 151 -#define TK_FAIL 152 -#define TK_FOR 153 -#define TK_IGNORE 154 -#define TK_IMMEDIATE 155 -#define TK_INITIALLY 156 -#define TK_INSTEAD 157 -#define TK_MATCH 158 -#define TK_KEY 159 -#define TK_OF 160 -#define TK_RAISE 161 -#define TK_REPLACE 162 -#define TK_RESTRICT 163 -#define TK_ROW 164 -#define TK_STATEMENT 165 -#define TK_TRIGGER 166 -#define TK_VIEW 167 -#define TK_COUNT 168 -#define TK_SUM 169 -#define TK_AVG 170 -#define TK_MIN 171 -#define TK_MAX 172 -#define TK_FIRST 173 -#define TK_LAST 174 -#define TK_TOP 175 -#define TK_BOTTOM 176 -#define TK_STDDEV 177 -#define TK_PERCENTILE 178 -#define TK_APERCENTILE 179 -#define TK_LEASTSQUARES 180 -#define TK_HISTOGRAM 181 -#define TK_DIFF 182 -#define TK_SPREAD 183 -#define TK_TWA 184 -#define TK_INTERP 185 -#define TK_LAST_ROW 186 -#define TK_RATE 187 -#define TK_IRATE 188 -#define TK_SUM_RATE 189 -#define TK_SUM_IRATE 190 -#define TK_AVG_RATE 191 -#define TK_AVG_IRATE 192 -#define TK_TBID 193 -#define TK_SEMI 194 -#define TK_NONE 195 -#define TK_PREV 196 -#define TK_LINEAR 197 -#define TK_IMPORT 198 -#define TK_METRIC 199 -#define TK_TBNAME 200 -#define TK_JOIN 201 -#define TK_METRICS 202 -#define TK_STABLE 203 -#define TK_INSERT 204 -#define TK_INTO 205 -#define TK_VALUES 206 +#define TK_CACHELAST 99 +#define TK_LP 100 +#define TK_RP 101 +#define TK_TAGS 102 +#define TK_USING 103 +#define TK_AS 104 +#define TK_COMMA 105 +#define TK_NULL 106 +#define TK_SELECT 107 +#define TK_UNION 108 +#define TK_ALL 109 +#define TK_FROM 110 +#define TK_VARIABLE 111 +#define TK_INTERVAL 112 +#define TK_FILL 113 +#define TK_SLIDING 114 +#define TK_ORDER 115 +#define TK_BY 116 +#define TK_ASC 117 +#define TK_DESC 118 +#define TK_GROUP 119 +#define TK_HAVING 120 +#define TK_LIMIT 121 +#define TK_OFFSET 122 +#define TK_SLIMIT 123 +#define TK_SOFFSET 124 +#define TK_WHERE 125 +#define TK_NOW 126 +#define TK_RESET 127 +#define TK_QUERY 128 +#define TK_ADD 129 +#define TK_COLUMN 130 +#define TK_TAG 131 +#define TK_CHANGE 132 +#define TK_SET 133 +#define TK_KILL 134 +#define TK_CONNECTION 135 +#define TK_STREAM 136 +#define TK_COLON 137 +#define TK_ABORT 138 +#define TK_AFTER 139 +#define TK_ATTACH 140 +#define TK_BEFORE 141 +#define TK_BEGIN 142 +#define TK_CASCADE 143 +#define TK_CLUSTER 144 +#define TK_CONFLICT 145 +#define TK_COPY 146 +#define TK_DEFERRED 147 +#define TK_DELIMITERS 148 +#define TK_DETACH 149 +#define TK_EACH 150 +#define TK_END 151 +#define TK_EXPLAIN 152 +#define TK_FAIL 153 +#define TK_FOR 154 +#define TK_IGNORE 155 +#define TK_IMMEDIATE 156 +#define TK_INITIALLY 157 +#define TK_INSTEAD 158 +#define TK_MATCH 159 +#define TK_KEY 160 +#define TK_OF 161 +#define TK_RAISE 162 +#define TK_REPLACE 163 +#define TK_RESTRICT 164 +#define TK_ROW 165 +#define TK_STATEMENT 166 +#define TK_TRIGGER 167 +#define TK_VIEW 168 +#define TK_COUNT 169 +#define TK_SUM 170 +#define TK_AVG 171 +#define TK_MIN 172 +#define TK_MAX 173 +#define TK_FIRST 174 +#define TK_LAST 175 +#define TK_TOP 176 +#define TK_BOTTOM 177 +#define TK_STDDEV 178 +#define TK_PERCENTILE 179 +#define TK_APERCENTILE 180 +#define TK_LEASTSQUARES 181 +#define TK_HISTOGRAM 182 +#define TK_DIFF 183 +#define TK_SPREAD 184 +#define TK_TWA 185 +#define TK_INTERP 186 +#define TK_LAST_ROW 187 +#define TK_RATE 188 +#define TK_IRATE 189 +#define TK_SUM_RATE 190 +#define TK_SUM_IRATE 191 +#define TK_AVG_RATE 192 +#define TK_AVG_IRATE 193 +#define TK_TBID 194 +#define TK_SEMI 195 +#define TK_NONE 196 +#define TK_PREV 197 +#define TK_LINEAR 198 +#define TK_IMPORT 199 +#define TK_METRIC 200 +#define TK_TBNAME 201 +#define TK_JOIN 202 +#define TK_METRICS 203 +#define TK_STABLE 204 +#define TK_INSERT 205 +#define TK_INTO 206 +#define TK_VALUES 207 #define TK_SPACE 300 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 7d5779c43f7c05ed7675cb59ef3036c14f852938..7f6a8d65e7c6153751da2525d09a0a40783e6e91 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -8,26 +8,27 @@ extern "C" { #include "taosdef.h" #define GET_TYPED_DATA(_v, _finalType, _type, _data) \ - switch (_type) { \ - case TSDB_DATA_TYPE_TINYINT: \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ (_v) = (_finalType)GET_INT8_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_SMALLINT: \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ (_v) = (_finalType)GET_INT16_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_BIGINT: \ + break; \ + case TSDB_DATA_TYPE_BIGINT: \ (_v) = (_finalType)(GET_INT64_VAL(_data)); \ - break; \ - case TSDB_DATA_TYPE_FLOAT: \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ (_v) = (_finalType)GET_FLOAT_VAL(_data); \ - break; \ - case TSDB_DATA_TYPE_DOUBLE: \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ (_v) = (_finalType)GET_DOUBLE_VAL(_data); \ - break; \ - default: \ + break; \ + default: \ (_v) = (_finalType)GET_INT32_VAL(_data); \ - break; \ - }; + break; \ + } #ifdef __cplusplus } diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 95f1d27b591c2503461826e175c3e768c0db77cc..cbe64484b11462fcf3f107625cb0e00b0ac682f4 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -48,7 +48,7 @@ typedef struct { void * pVnode; SRpcMsg rpcMsg; SRspRet rspRet; - char reserveForSync[16]; + char reserveForSync[24]; SWalHead pHead[]; } SVWriteMsg; diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf73988ab25db7544b9a52215d2279630c63..bf5278430054598b31e591ff728412556dabc6f0 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -3,4 +3,5 @@ PROJECT(TDengine) ADD_SUBDIRECTORY(shell) ADD_SUBDIRECTORY(taosdemo) +ADD_SUBDIRECTORY(taosdemox) ADD_SUBDIRECTORY(taosdump) diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 7e5ebb059667c5fc4d530f1fab3957a962ed3fda..24156956174364ae1be6abfa2af4c3db7c0db712 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -86,6 +86,6 @@ extern void set_terminal_mode(); extern int get_old_terminal_mode(struct termios* tio); extern void reset_terminal_mode(); extern SShellArguments args; -extern TAOS_RES* result; +extern int64_t result; #endif diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 7a9e242668822cd4383050fa2dcebe248024f9e7..fca0e93472a350e8a6fddb8acc5ff54af67fa833 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -46,7 +46,7 @@ char CONTINUE_PROMPT[] = " -> "; int prompt_size = 6; #endif -TAOS_RES *result = NULL; +int64_t result = 0; SShellHistory history; #define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 @@ -260,6 +260,14 @@ int32_t shellRunCommand(TAOS* con, char* command) { } +void freeResultWithRid(int64_t rid) { + SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); + if(pSql){ + taos_free_result(pSql); + taosReleaseRef(tscObjRef, rid); + } +} + void shellRunCommandOnServer(TAOS *con, char command[]) { int64_t st, et; wordexp_t full_path; @@ -300,12 +308,14 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { return; } + int64_t oresult = atomic_load_64(&result); + if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - atomic_store_ptr(&result, 0); - taos_free_result(pSql); + atomic_store_64(&result, 0); + freeResultWithRid(oresult); return; } @@ -313,8 +323,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - atomic_store_ptr(&result, 0); - taos_free_result(pSql); + atomic_store_64(&result, 0); + freeResultWithRid(oresult); return; } @@ -336,8 +346,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { wordfree(&full_path); } - atomic_store_ptr(&result, 0); - taos_free_result(pSql); + atomic_store_64(&result, 0); + freeResultWithRid(oresult); } /* Function to do regular expression check */ @@ -501,7 +511,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { row = taos_fetch_row(tres); } while( row != NULL); - result = NULL; + result = 0; fclose(fp); return numOfRows; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 15b2b077c9cf96e4c8a430a1b4c32dabb8cc2913..9eb30ccdccc60d9b975236036b7f3aab0675b196 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -33,7 +33,7 @@ const char *argp_program_bug_address = ""; static char doc[] = ""; static char args_doc[] = ""; static struct argp_option options[] = { - {"host", 'h', "HOST", 0, "TDengine server IP address to connect. The default host is localhost."}, + {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, {"password", 'p', "PASSWORD", OPTION_ARG_OPTIONAL, "The password to use when connecting to the server."}, {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 4f0c5e3f9938b2d3c5281a902e4dbc44c221bf82..041ad71ccba47cf2b84984af89d25ff15ccf16ce 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -19,15 +19,31 @@ #include "tnettest.h" pthread_t pid; +static tsem_t cancelSem; void shellQueryInterruptHandler(int signum) { + tsem_post(&cancelSem); +} + +void *cancelHandler(void *arg) { + while(1) { + if (tsem_wait(&cancelSem) != 0) { + taosMsleep(10); + continue; + } + #ifdef LINUX - void* pResHandle = atomic_val_compare_exchange_64(&result, result, 0); - taos_stop_query(pResHandle); + int64_t rid = atomic_val_compare_exchange_64(&result, result, 0); + SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); + taos_stop_query(pSql); + taosReleaseRef(tscObjRef, rid); #else - printf("\nReceive ctrl+c or other signal, quit shell.\n"); - exit(0); + printf("\nReceive ctrl+c or other signal, quit shell.\n"); + exit(0); #endif + } + + return NULL; } int checkVersion() { @@ -105,6 +121,14 @@ int main(int argc, char* argv[]) { exit(EXIT_FAILURE); } + if (tsem_init(&cancelSem, 0, 0) != 0) { + printf("failed to create cancel semphore\n"); + exit(EXIT_FAILURE); + } + + pthread_t spid; + pthread_create(&spid, NULL, cancelHandler, NULL); + /* Interrupt handler. */ struct sigaction act; memset(&act, 0, sizeof(struct sigaction)); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 64eed9eace33034cc7182b47182f5f7798852859..7cb6c75302a0d2417469c5571c88fc382c6919ea 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -24,7 +24,7 @@ void printHelp() { printf("taos shell is used to test the TDengine database\n"); printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost."); + printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); printf("%s%s\n", indent, "-P"); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 53e7d2398450fe22a11da1e7254e0c2ab5f02ea4..1d77a6bb6395d7b171a3d79ee75df27ec1826f34 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -95,7 +95,7 @@ typedef struct DemoArguments { {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, #endif {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, + {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, {0, 's', "sql file", 0, "The select sql file.", 3}, {0, 'M', 0, 0, "Use metric flag.", 13}, @@ -205,10 +205,10 @@ typedef struct DemoArguments { arguments->tb_prefix = arg; break; case 'M': - arguments->use_metric = false; + arguments->use_metric = true; break; case 'x': - arguments->insert_only = false; + arguments->insert_only = true; break; case 'c': if (wordexp(arg, &full_path, 0) != 0) { @@ -406,9 +406,9 @@ typedef struct DemoArguments { } else if (strcmp(argv[i], "-m") == 0) { arguments->tb_prefix = argv[++i]; } else if (strcmp(argv[i], "-M") == 0) { - arguments->use_metric = false; + arguments->use_metric = true; } else if (strcmp(argv[i], "-x") == 0) { - arguments->insert_only = false; + arguments->insert_only = true; } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-O") == 0) { @@ -476,6 +476,14 @@ typedef struct { int notFinished; tsem_t lock_sem; int counter; + + // insert delay statitics + int64_t cntDelay; + int64_t totalDelay; + int64_t avgDelay; + int64_t maxDelay; + int64_t minDelay; + } info; typedef struct { @@ -575,7 +583,7 @@ int main(int argc, char *argv[]) { arguments.num_of_DPT = 100000; arguments.num_of_RPR = 1000; arguments.use_metric = true; - arguments.insert_only = true; + arguments.insert_only = false; // end change parse_args(argc, argv, &arguments); @@ -739,6 +747,9 @@ int main(int argc, char *argv[]) { printf("Inserting data......\n"); pthread_t *pids = malloc(threads * sizeof(pthread_t)); info *infos = malloc(threads * sizeof(info)); + + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(info)); int a = ntables / threads; if (a < 1) { @@ -768,6 +779,7 @@ int main(int argc, char *argv[]) { t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; t_info->counter = 0; + t_info->minDelay = INT16_MAX; tsem_init(&(t_info->mutex_sem), 0, 1); t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; @@ -799,12 +811,29 @@ int main(int argc, char *argv[]) { t, (int64_t)ntables * nrecords_per_table, nrecords_per_request, (int64_t)ntables * nrecords_per_table / t); + int64_t totalDelay = 0; + int64_t maxDelay = 0; + int64_t minDelay = INT16_MAX; + int64_t cntDelay = 0; + double avgDelay = 0; for (int i = 0; i < threads; i++) { info *t_info = infos + i; taos_close(t_info->taos); tsem_destroy(&(t_info->mutex_sem)); tsem_destroy(&(t_info->lock_sem)); + + totalDelay += t_info->totalDelay; + cntDelay += t_info->cntDelay; + if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; + if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; } + avgDelay = (double)totalDelay / cntDelay; + + fprintf(fp, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + + printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); free(pids); free(infos); @@ -859,7 +888,7 @@ int main(int argc, char *argv[]) { } - if (!insert_only) { + if (false == insert_only) { // query data pthread_t read_id; info *rInfo = malloc(sizeof(info)); @@ -998,7 +1027,7 @@ void * createTable(void *sarg) /* Create all the tables; */ printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols); + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s);", winfo->db_name, winfo->tb_prefix, i, winfo->cols); queryDB(winfo->taos, command); } } else { @@ -1204,6 +1233,41 @@ void *readMetric(void *sarg) { return NULL; } +static int queryDbExec(TAOS *taos, char *command, int type) { + int i; + TAOS_RES *res = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } + + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + if (1 == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + void queryDB(TAOS *taos, char *command) { int i; TAOS_RES *pSql = NULL; @@ -1273,7 +1337,21 @@ void *syncWrite(void *sarg) { } /* puts(buffer); */ - queryDB(winfo->taos, buffer); + int64_t startTs; + int64_t endTs; + startTs = taosGetTimestampUs(); + //queryDB(winfo->taos, buffer); + int affectedRows = queryDbExec(winfo->taos, buffer, 1); + + if (0 <= affectedRows){ + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + if (delay > winfo->maxDelay) winfo->maxDelay = delay; + if (delay < winfo->minDelay) winfo->minDelay = delay; + winfo->cntDelay++; + winfo->totalDelay += delay; + //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; + } if (tID == winfo->end_table_id) { i = inserted; diff --git a/src/kit/taosdemox/CMakeLists.txt b/src/kit/taosdemox/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..3f5e725aeaf9fa957e8aa9cdd54978df7fbd94ed --- /dev/null +++ b/src/kit/taosdemox/CMakeLists.txt @@ -0,0 +1,25 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/include) + +IF (TD_LINUX) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(taosdemox ${SRC}) + + #find_program(HAVE_CURL NAMES curl) + IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) + ADD_DEFINITIONS(-DTD_LOWA_CURL) + LINK_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/lib) + ADD_LIBRARY(curl STATIC IMPORTED) + SET_PROPERTY(TARGET curl PROPERTY IMPORTED_LOCATION ${TD_COMMUNITY_DIR}/deps/libcurl/lib/libcurl.a) + TARGET_LINK_LIBRARIES(taosdemox curl) + ENDIF () + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemox taos_static cJson) + ELSE () + TARGET_LINK_LIBRARIES(taosdemox taos cJson) + ENDIF () +ENDIF () diff --git a/src/kit/taosdemox/insert.json b/src/kit/taosdemox/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..88416c13a4cad68a25d090f69f9b3ae7dee66296 --- /dev/null +++ b/src/kit/taosdemox/insert.json @@ -0,0 +1,53 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 2, + "result_file": "./insert_res.txt", + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 2, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 1, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/src/kit/taosdemox/query.json b/src/kit/taosdemox/query.json new file mode 100644 index 0000000000000000000000000000000000000000..53d0b319212196257aa3e84be1221bd6e2bd0d8d --- /dev/null +++ b/src/kit/taosdemox/query.json @@ -0,0 +1,17 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db01", + "super_table_query": + {"rate":1, "concurrent":1, + "sqls": [{"sql": "select count(*) from stb01", "result": "./query_res0.txt"}] + }, + "sub_table_query": + {"stblname": "stb01", "rate":1, "threads":1, + "sqls": [{"sql": "select count(*) from xxxx", "result": "./query_res1.txt"}] + } +} diff --git a/src/kit/taosdemox/subscribe.json b/src/kit/taosdemox/subscribe.json new file mode 100644 index 0000000000000000000000000000000000000000..6dfacdd6ed112b398cf38731147cafc02879efe2 --- /dev/null +++ b/src/kit/taosdemox/subscribe.json @@ -0,0 +1,17 @@ +{ + "filetype":"subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db01", + "super_table_query": + {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", + "sqls": [{"sql": "select avg(c1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] + }, + "sub_table_query": + {"stblname": "stb01", "threads":1, "mode":"sync", "interval":10000, "restart":"yes", "keepProgress":"yes", + "sqls": [{"sql": "select col1 from xxxx where col1 > 10;", "result": "./subscribe_res1.txt"}] + } +} diff --git a/src/kit/taosdemox/taosdemox.c b/src/kit/taosdemox/taosdemox.c new file mode 100644 index 0000000000000000000000000000000000000000..5c9fd025f0a39e03ea8eaedc1bc81bc7e88d8eb4 --- /dev/null +++ b/src/kit/taosdemox/taosdemox.c @@ -0,0 +1,4621 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +/* + when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread. +*/ + +#define _GNU_SOURCE +#define CURL_STATICLIB + +#ifdef TD_LOWA_CURL +#include "curl/curl.h" +#endif + +#ifdef LINUX + #include "os.h" + #include "cJSON.h" + #include + #include + #include + #ifndef _ALPINE + #include + #endif + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include +#else + #include + #include + #include + #include "os.h" + + #pragma comment ( lib, "libcurl.lib" ) + #pragma comment ( lib, "ws2_32.lib" ) + #pragma comment ( lib, "winmm.lib" ) + #pragma comment ( lib, "wldap32.lib" ) +#endif + +#include "taos.h" +#include "tutil.h" + +extern char configDir[]; + +#define INSERT_JSON_NAME "insert.json" +#define QUERY_JSON_NAME "query.json" +#define SUBSCRIBE_JSON_NAME "subscribe.json" + +#define INSERT_MODE 0 +#define QUERY_MODE 1 +#define SUBSCRIBE_MODE 2 + +#define MAX_SQL_SIZE 65536 +#define BUFFER_SIZE (65536*2) +#define MAX_DB_NAME_SIZE 64 +#define MAX_TB_NAME_SIZE 64 +#define MAX_DATA_SIZE 16000 +#define MAX_NUM_DATATYPE 10 +#define OPT_ABORT 1 /* –abort */ +#define STRING_LEN 60000 +#define MAX_PREPARED_RAND 1000000 +//#define MAX_SQL_SIZE 65536 +#define MAX_FILE_NAME_LEN 256 + +#define MAX_SAMPLES_ONCE_FROM_FILE 10000 +#define MAX_NUM_DATATYPE 10 + +#define MAX_DB_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 8 +#define MAX_COLUMN_COUNT 1024 +#define MAX_TAG_COUNT 128 + +#define MAX_QUERY_SQL_COUNT 10 +#define MAX_QUERY_SQL_LENGTH 256 + + +#define MAX_LINE_COUNT_IN_MEM 10000 + +typedef enum CREATE_SUB_TALBE_MOD_EN { + PRE_CREATE_SUBTBL, + AUTO_CREATE_SUBTBL, + NO_CREATE_SUBTBL +} CREATE_SUB_TALBE_MOD_EN; + +typedef enum TALBE_EXISTS_EN { + TBL_ALREADY_EXISTS, + TBL_NO_EXISTS, + TBL_EXISTS_BUTT +} TALBE_EXISTS_EN; + +enum MODE { + SYNC, + ASYNC, + MODE_BUT +}; + +enum QUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} ; + +enum _describe_table_index { + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC +}; + +typedef struct { + char field[TSDB_COL_NAME_LEN + 1]; + char type[16]; + int length; + char note[128]; +} SColDes; + +/* Used by main to communicate with parse_opt. */ +typedef struct SArguments_S { + char * metaFile; + char * host; + uint16_t port; + char * user; + char * password; + char * database; + int replica; + char * tb_prefix; + char * sqlFile; + bool use_metric; + bool insert_only; + char * output_file; + int mode; + char * datatype[MAX_NUM_DATATYPE + 1]; + int len_of_binary; + int num_of_CPR; + int num_of_threads; + int num_of_RPR; + int num_of_tables; + int num_of_DPT; + int abort; + int disorderRatio; + int disorderRange; + int method_of_delete; + char ** arg_list; +} SArguments; + +typedef struct SColumn_S { + char field[TSDB_COL_NAME_LEN + 1]; + char dataType[MAX_TB_NAME_SIZE]; + int dataLen; + char note[128]; +} StrColumn; + +typedef struct SSuperTable_S { + char sTblName[MAX_TB_NAME_SIZE]; + int childTblCount; + bool superTblExists; // 0: no, 1: yes + bool childTblExists; // 0: no, 1: yes + int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table + char childTblPrefix[MAX_TB_NAME_SIZE]; + char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample + char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful + int insertRate; // 0: unlimit > 0 rows/s + + int multiThreadWriteOneTbl; // 0: no, 1: yes + int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl + int rowsPerTbl; // + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms or us by database precision + int maxSqlLen; // + + int64_t insertRows; // 0: no limit + int timeStampStep; + char startTimestamp[MAX_TB_NAME_SIZE]; // + char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFile[MAX_FILE_NAME_LEN]; + char tagsFile[MAX_FILE_NAME_LEN]; + + int columnCount; + StrColumn columns[MAX_COLUMN_COUNT]; + int tagCount; + StrColumn tags[MAX_TAG_COUNT]; + + char* childTblName; + char* colsOfCreatChildTable; + int lenOfOneRow; + int lenOfTagOfOneRow; + + char* sampleDataBuf; + int sampleDataBufSize; + //int sampleRowCount; + //int sampleUsePos; + + int tagSource; // 0: rand, 1: tag sample + char* tagDataBuf; + int tagSampleCount; + int tagUsePos; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; +} SSuperTable; + +typedef struct SDbCfg_S { +// int maxtablesPerVnode; + int minRows; + int maxRows; + int comp; + int walLevel; + int fsync; + int replica; + int update; + int keep; + int days; + int cache; + int blocks; + int quorum; + char precision[MAX_TB_NAME_SIZE]; +} SDbCfg; + +typedef struct SDataBase_S { + char dbName[MAX_DB_NAME_SIZE]; + int drop; // 0: use exists, 1: if exists, drop then new create + SDbCfg dbCfg; + int superTblCount; + SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; +} SDataBase; + +typedef struct SDbs_S { + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_DB_NAME_SIZE]; + uint16_t port; + char user[MAX_DB_NAME_SIZE]; + char password[MAX_DB_NAME_SIZE]; + char resultFile[MAX_FILE_NAME_LEN]; + bool use_metric; + bool insert_only; + bool do_aggreFunc; + bool queryMode; + + int threadCount; + int dbCount; + SDataBase db[MAX_DB_COUNT]; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; +} SDbs; + +typedef struct SuperQueryInfo_S { + int rate; // 0: unlimit > 0 loop/s + int concurrent; + int sqlCount; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int subscribeRestart; + int subscribeKeepProgress; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; +} SuperQueryInfo; + +typedef struct SubQueryInfo_S { + char sTblName[MAX_TB_NAME_SIZE]; + int rate; // 0: unlimit > 0 loop/s + int threadCnt; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int subscribeRestart; + int subscribeKeepProgress; + int childTblCount; + char childTblPrefix[MAX_TB_NAME_SIZE]; + int sqlCount; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + + char* childTblName; +} SubQueryInfo; + +typedef struct SQueryMetaInfo_S { + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_DB_NAME_SIZE]; + uint16_t port; + char user[MAX_DB_NAME_SIZE]; + char password[MAX_DB_NAME_SIZE]; + char dbName[MAX_DB_NAME_SIZE]; + char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful + + SuperQueryInfo superQueryInfo; + SubQueryInfo subQueryInfo; +} SQueryMetaInfo; + +typedef struct SThreadInfo_S { + TAOS *taos; + #ifdef TD_LOWA_CURL + CURL *curl_handle; + #endif + int threadID; + char db_name[MAX_DB_NAME_SIZE]; + char fp[4096]; + char tb_prefix[MAX_TB_NAME_SIZE]; + int start_table_id; + int end_table_id; + int data_of_rate; + int64_t start_time; + char* cols; + bool use_metric; + SSuperTable* superTblInfo; + + // for async insert + tsem_t lock_sem; + int64_t counter; + int64_t st; + int64_t et; + int64_t lastTs; + int nrecords_per_request; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; +} threadInfo; + +typedef struct curlMemInfo_S { + char *buf; + size_t sizeleft; + } curlMemInfo; + + + +#ifdef LINUX + /* The options we understand. */ + static struct argp_option options[] = { + {0, 'f', "meta file", 0, "The meta data to the execution procedure, if use -f, all others options invalid. Default is NULL.", 0}, + #ifdef _TD_POWER_ + {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/power/'.", 1}, + {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'powerdb'.", 2}, + #else + {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 1}, + {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 2}, + #endif + {0, 'h', "host", 0, "The host to connect to TDengine. Default is localhost.", 2}, + {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 2}, + {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, + {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, + {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 4}, + {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 4}, + {0, 's', "sql file", 0, "The select sql file.", 6}, + {0, 'M', 0, 0, "Use metric flag.", 4}, + {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 6}, + {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 4}, + {0, 'b', "type_of_cols", 0, "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.", 4}, + {0, 'w', "length_of_chartype", 0, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16", 4}, + {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 10.", 4}, + {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 4}, + // {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 100.", 4}, + {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 4}, + {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 10000.", 4}, + {0, 'x', 0, 0, "Not insert only flag.", 4}, + {0, 'O', "disorderRatio", 0, "Insert mode--0: In order, > 0: disorder ratio. Default is in order.", 4}, + {0, 'R', "disorderRang", 0, "Out of order data's range, ms, default is 1000.", 4}, + //{0, 'D', "delete database", 0, "if elete database if exists. 0: no, 1: yes, default is 1", 5}, + {0}}; + +/* Parse a single option. */ +static error_t parse_opt(int key, char *arg, struct argp_state *state) { + // Get the input argument from argp_parse, which we know is a pointer to our arguments structure. + SArguments *arguments = state->input; + wordexp_t full_path; + char **sptr; + switch (key) { + case 'f': + arguments->metaFile = arg; + break; + case 'h': + arguments->host = arg; + break; + case 'p': + arguments->port = atoi(arg); + break; + case 'u': + arguments->user = arg; + break; + case 'P': + arguments->password = arg; + break; + case 'o': + arguments->output_file = arg; + break; + case 's': + arguments->sqlFile = arg; + break; + case 'q': + arguments->mode = atoi(arg); + break; + case 'T': + arguments->num_of_threads = atoi(arg); + break; + //case 'r': + // arguments->num_of_RPR = atoi(arg); + // break; + case 't': + arguments->num_of_tables = atoi(arg); + break; + case 'n': + arguments->num_of_DPT = atoi(arg); + break; + case 'd': + arguments->database = arg; + break; + case 'l': + arguments->num_of_CPR = atoi(arg); + break; + case 'b': + sptr = arguments->datatype; + if (strstr(arg, ",") == NULL) { + if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && + strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && + strcasecmp(arg, "SMALLINT") != 0 && strcasecmp(arg, "TIMESTAMP") != 0 && + strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && + strcasecmp(arg, "BINARY") != 0 && strcasecmp(arg, "NCHAR") != 0) { + argp_error(state, "Invalid data_type!"); + } + sptr[0] = arg; + } else { + int index = 0; + char *dupstr = strdup(arg); + char *running = dupstr; + char *token = strsep(&running, ","); + while (token != NULL) { + if (strcasecmp(token, "INT") != 0 && strcasecmp(token, "FLOAT") != 0 && + strcasecmp(token, "TINYINT") != 0 && strcasecmp(token, "BOOL") != 0 && + strcasecmp(token, "SMALLINT") != 0 && strcasecmp(token, "TIMESTAMP") != 0 && + strcasecmp(token, "BIGINT") != 0 && strcasecmp(token, "DOUBLE") != 0 && + strcasecmp(token, "BINARY") != 0 && strcasecmp(token, "NCHAR") != 0) { + argp_error(state, "Invalid data_type!"); + } + sptr[index++] = token; + token = strsep(&running, ","); + if (index >= MAX_NUM_DATATYPE) break; + } + } + break; + case 'w': + arguments->len_of_binary = atoi(arg); + break; + case 'm': + arguments->tb_prefix = arg; + break; + case 'M': + arguments->use_metric = true; + break; + case 'x': + arguments->insert_only = false; + break; + case 'c': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", arg); + return -1; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + break; + case 'O': + arguments->disorderRatio = atoi(arg); + if (arguments->disorderRatio < 0 || arguments->disorderRatio > 100) + { + argp_error(state, "Invalid disorder ratio, should 1 ~ 100!"); + } + break; + case 'R': + arguments->disorderRange = atoi(arg); + break; + case 'a': + arguments->replica = atoi(arg); + if (arguments->replica > 3 || arguments->replica < 1) + { + arguments->replica = 1; + } + break; + //case 'D': + // arguments->method_of_delete = atoi(arg); + // break; + case OPT_ABORT: + arguments->abort = 1; + break; + case ARGP_KEY_ARG: + /*arguments->arg_list = &state->argv[state->next-1]; + state->next = state->argc;*/ + argp_usage(state); + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static struct argp argp = {options, parse_opt, 0, 0}; + +void parse_args(int argc, char *argv[], SArguments *arguments) { + argp_parse(&argp, argc, argv, 0, 0, arguments); + if (arguments->abort) { + #ifndef _ALPINE + error(10, 0, "ABORTED"); + #else + abort(); + #endif + } +} + +#else + void printHelp() { + char indent[10] = " "; + printf("%s%s\n", indent, "-f"); + printf("%s%s%s\n", indent, indent, "The meta file to the execution procedure. Default is './meta.json'."); + printf("%s%s\n", indent, "-c"); + printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); + } + + void parse_args(int argc, char *argv[], SArguments *arguments) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-f") == 0) { + arguments->metaFile = argv[++i]; + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "wrong options\n"); + printHelp(); + exit(EXIT_FAILURE); + } + } + } +#endif + +static bool getInfoFromJsonFile(char* file); +//static int generateOneRowDataForStb(SSuperTable* stbInfo); +//static int getDataIntoMemForStb(SSuperTable* stbInfo); +static void init_rand_data(); +static int createDatabases(); +static void createChildTables(); +static int queryDbExec(TAOS *taos, char *command, int type); + +/* ************ Global variables ************ */ + +int32_t randint[MAX_PREPARED_RAND]; +int64_t randbigint[MAX_PREPARED_RAND]; +float randfloat[MAX_PREPARED_RAND]; +double randdouble[MAX_PREPARED_RAND]; +char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; + +SArguments g_args = {NULL, + "127.0.0.1", // host + 6030, // port + "root", // user + #ifdef _TD_POWER_ + "powerdb", // password + #else + "taosdata", // password + #endif + "test", // database + 1, // replica + "t", // tb_prefix + NULL, // sqlFile + false, // use_metric + true, // insert_only + "./output.txt", // output_file + 0, // mode : sync or async + { + "TINYINT", // datatype + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "BINARY", + "NCHAR", + "BOOL", + "TIMESTAMP" + }, + 16, // len_of_binary + 10, // num_of_CPR + 10, // num_of_connections/thread + 100, // num_of_RPR + 10000, // num_of_tables + 10000, // num_of_DPT + 0, // abort + 0, // disorderRatio + 1000, // disorderRange + 1, // method_of_delete + NULL // arg_list +}; + + +static int g_jsonType = 0; +static SDbs g_Dbs; +static int g_totalChildTables = 0; +static SQueryMetaInfo g_queryInfo; +static FILE * g_fpOfInsertResult = NULL; + + +void tmfclose(FILE *fp) { + if (NULL != fp) { + fclose(fp); + } +} + +void tmfree(char *buf) { + if (NULL != buf) { + free(buf); + } +} + +static int queryDbExec(TAOS *taos, char *command, int type) { + int i; + TAOS_RES *res = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } + + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + +static void getResult(TAOS_RES *res, char* resultFileName) { + TAOS_ROW row = NULL; + int num_rows = 0; + int num_fields = taos_field_count(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + + FILE *fp = NULL; + if (resultFileName[0] != 0) { + fp = fopen(resultFileName, "at"); + if (fp == NULL) { + fprintf(stderr, "failed to open result file: %s, result will not save to file\n", resultFileName); + } + } + + char* databuf = (char*) calloc(1, 100*1024*1024); + if (databuf == NULL) { + fprintf(stderr, "failed to malloc, warning: save result to file slowly!\n"); + return ; + } + + int totalLen = 0; + char temp[16000]; + + // fetch the records row by row + while ((row = taos_fetch_row(res))) { + if (totalLen >= 100*1024*1024 - 32000) { + if (fp) fprintf(fp, "%s", databuf); + totalLen = 0; + memset(databuf, 0, 100*1024*1024); + } + num_rows++; + int len = taos_print_row(temp, row, fields, num_fields); + len += sprintf(temp + len, "\n"); + //printf("query result:%s\n", temp); + memcpy(databuf + totalLen, temp, len); + totalLen += len; + } + + if (fp) fprintf(fp, "%s", databuf); + tmfclose(fp); + free(databuf); +} + +static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { + TAOS_RES *res = taos_query(taos, command); + if (res == NULL || taos_errno(res) != 0) { + printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res)); + taos_free_result(res); + return; + } + + getResult(res, resultFileName); + taos_free_result(res); +} + +double getCurrentTime() { + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { + perror("Failed to get current time in ms"); + return 0.0; + } + + return tv.tv_sec + tv.tv_usec / 1E6; +} + +static int32_t rand_bool(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 2; +} + +static int32_t rand_tinyint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 128; +} + +static int32_t rand_smallint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 32767; +} + +static int32_t rand_int(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor]; +} + +static int64_t rand_bigint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randbigint[cursor]; + +} + +static float rand_float(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randfloat[cursor]; +} + +static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; +void rand_string(char *str, int size) { + str[0] = 0; + if (size > 0) { + //--size; + int n; + for (n = 0; n < size; n++) { + int key = rand_tinyint() % (int)(sizeof(charset) - 1); + str[n] = charset[key]; + } + str[n] = 0; + } +} + +static double rand_double() { + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randdouble[cursor]; + +} + +static void init_rand_data() { + for (int i = 0; i < MAX_PREPARED_RAND; i++){ + randint[i] = (int)(rand() % 65535); + randbigint[i] = (int64_t)(rand() % 2147483648); + randfloat[i] = (float)(rand() / 1000.0); + randdouble[i] = (double)(rand() / 1000000.0); + } +} + +static void printfInsertMeta() { + printf("\033[1m\033[40;32m================ insert.json parse result START ================\033[0m\n"); + printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("user: \033[33m%s\033[0m\n", g_Dbs.user); + printf("password: \033[33m%s\033[0m\n", g_Dbs.password); + printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); + printf("thread count: \033[33m%d\033[0m\n", g_Dbs.threadCount); + + printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + printf("database[\033[33m%d\033[0m]:\n", i); + printf(" database name: \033[33m%s\033[0m\n", g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + printf(" drop: \033[33mno\033[0m\n"); + }else { + printf(" drop: \033[33myes\033[0m\n"); + } + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); + } else { + printf(" precision error: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); + exit(EXIT_FAILURE); + } + } + + printf(" super table count: \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + printf(" super table[\033[33m%d\033[0m]:\n", j); + + printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" childTblExists: \033[33m%s\033[0m\n", "error"); + } + + printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); + printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); + printf(" insertRate: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].insertRate); + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); + }else { + printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); + } + printf(" numberOfTblInOneSql: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); + printf(" rowsPerTbl: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); + printf(" disorderRange: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRange); + printf(" disorderRatio: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRatio); + printf(" maxSqlLen: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + printf(" timeStampStep: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].timeStampStep); + printf(" startTimestamp: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].startTimestamp); + printf(" sampleFormat: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFormat); + printf(" sampleFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFile); + printf(" tagsFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].tagsFile); + + printf(" columnCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + printf("column[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + printf("\n"); + + printf(" tagCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + printf("tag[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + printf("\n"); + } + printf("\n"); + } + printf("\033[1m\033[40;32m================ insert.json parse result END================\033[0m\n"); +} + +static void printfInsertMetaToFile(FILE* fp) { + fprintf(fp, "================ insert.json parse result START================\n"); + fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); + fprintf(fp, "user: %s\n", g_Dbs.user); + fprintf(fp, "password: %s\n", g_Dbs.password); + fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); + fprintf(fp, "thread count: %d\n", g_Dbs.threadCount); + + fprintf(fp, "database count: %d\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + fprintf(fp, "database[%d]:\n", i); + fprintf(fp, " database name: %s\n", g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + fprintf(fp, " drop: no\n"); + }else { + fprintf(fp, " drop: yes\n"); + } + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + fprintf(fp, " blocks: %d\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + fprintf(fp, " cache: %d\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + fprintf(fp, " days: %d\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + fprintf(fp, " keep: %d\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + fprintf(fp, " replica: %d\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + fprintf(fp, " update: %d\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + fprintf(fp, " minRows: %d\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + fprintf(fp, " maxRows: %d\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + fprintf(fp, " comp: %d\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + fprintf(fp, " walLevel: %d\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + fprintf(fp, " fsync: %d\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + fprintf(fp, " quorum: %d\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + } else { + fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + } + } + + fprintf(fp, " super table count: %d\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + fprintf(fp, " super table[%d]:\n", j); + + fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "yes"); + } else { + fprintf(fp, " autoCreateTable: %s\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "yes"); + } else { + fprintf(fp, " childTblExists: %s\n", "error"); + } + + fprintf(fp, " childTblCount: %d\n", g_Dbs.db[i].superTbls[j].childTblCount); + fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); + fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); + fprintf(fp, " insertRate: %d\n", g_Dbs.db[i].superTbls[j].insertRate); + fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + fprintf(fp, " multiThreadWriteOneTbl: no\n"); + }else { + fprintf(fp, " multiThreadWriteOneTbl: yes\n"); + } + fprintf(fp, " numberOfTblInOneSql: %d\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); + fprintf(fp, " rowsPerTbl: %d\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); + fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); + fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); + fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + + fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + fprintf(fp, "\n"); + + fprintf(fp, " tagCount: %d\n ", g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + fprintf(fp, "\n"); + } + fprintf(fp, "\n"); + } + fprintf(fp, "================ insert.json parse result END ================\n\n"); +} + +static void printfQueryMeta() { + printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); + printf("host: \033[33m%s:%u\033[0m\n", g_queryInfo.host, g_queryInfo.port); + printf("user: \033[33m%s\033[0m\n", g_queryInfo.user); + printf("password: \033[33m%s\033[0m\n", g_queryInfo.password); + printf("database name: \033[33m%s\033[0m\n", g_queryInfo.dbName); + + printf("\n"); + printf("super table query info: \n"); + printf("rate: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent); + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); + + if (SUBSCRIBE_MODE == g_jsonType) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); + } + printf("\n"); + printf("sub table query info: \n"); + printf("rate: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.rate); + printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.threadCnt); + printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount); + printf("childTblPrefix: \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.childTblPrefix); + + if (SUBSCRIBE_MODE == g_jsonType) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeKeepProgress); + } + + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.sqlCount); + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]); + } + printf("\n"); + printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); +} + +#ifdef TD_LOWA_CURL +static size_t responseCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + curlMemInfo* mem = (curlMemInfo*)userp; + + char *ptr = realloc(mem->buf, mem->sizeleft + realsize + 1); + if(ptr == NULL) { + /* out of memory! */ + printf("not enough memory (realloc returned NULL)\n"); + return 0; + } + + mem->buf = ptr; + memcpy(&(mem->buf[mem->sizeleft]), contents, realsize); + mem->sizeleft += realsize; + mem->buf[mem->sizeleft] = 0; + + //printf("result:%s\n\n", mem->buf); + + return realsize; +} + +void curlProceLogin(void) +{ + CURL *curl_handle; + CURLcode res; + + curlMemInfo chunk; + + chunk.buf = malloc(1); /* will be grown as needed by the realloc above */ + chunk.sizeleft = 0; /* no data at this point */ + + //curl_global_init(CURL_GLOBAL_ALL); + + /* init the curl session */ + curl_handle = curl_easy_init(); + + curl_easy_setopt(curl_handle,CURLOPT_POSTFIELDS,""); + curl_easy_setopt(curl_handle, CURLOPT_POST, 1); + + char dstUrl[128] = {0}; + snprintf(dstUrl, 128, "http://%s:6041/rest/login/root/taosdata", g_Dbs.host); + + /* specify URL to get */ + curl_easy_setopt(curl_handle, CURLOPT_URL, dstUrl); + + /* send all data to this function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, responseCallback); + + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk); + + /* do it! */ + res = curl_easy_perform(curl_handle); + + /* check for errors */ + if(res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + } + else { + //printf("response len:%lu, content: %s \n", (unsigned long)chunk.sizeleft, chunk.buf); + ; + } + + /* cleanup curl stuff */ + curl_easy_cleanup(curl_handle); + + free(chunk.buf); + + /* we're done with libcurl, so clean it up */ + //curl_global_cleanup(); + + return; +} + +int curlProceSql(char* host, uint16_t port, char* sqlstr, CURL *curl_handle) +{ + //curlProceLogin(); + + //CURL *curl_handle; + CURLcode res; + + curlMemInfo chunk; + + chunk.buf = malloc(1); /* will be grown as needed by the realloc above */ + chunk.sizeleft = 0; /* no data at this point */ + + + char dstUrl[128] = {0}; + snprintf(dstUrl, 128, "http://%s:%u/rest/sql", host, port+TSDB_PORT_HTTP); + + //curl_global_init(CURL_GLOBAL_ALL); + + /* init the curl session */ + //curl_handle = curl_easy_init(); + + //curl_easy_setopt(curl_handle,CURLOPT_POSTFIELDS,""); + curl_easy_setopt(curl_handle, CURLOPT_POST, 1L); + + /* specify URL to get */ + curl_easy_setopt(curl_handle, CURLOPT_URL, dstUrl); + + /* enable TCP keep-alive for this transfer */ + curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPALIVE, 1L); + /* keep-alive idle time to 120 seconds */ + curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPIDLE, 120L); + /* interval time between keep-alive probes: 60 seconds */ + curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPINTVL, 60L); + + /* send all data to this function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, responseCallback); + + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk); + + struct curl_slist *list = NULL; + list = curl_slist_append(list, "Authorization: Basic cm9vdDp0YW9zZGF0YQ=="); + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); + + /* Set the expected upload size. */ + curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)strlen(sqlstr)); + curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, sqlstr); + + /* get it! */ + res = curl_easy_perform(curl_handle); + + /* check for errors */ + if(res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + return -1; + } + else { + /* curl_easy_perform() block end and return result */ + //printf("[%32.32s] sql response len:%lu, content: %s \n\n", sqlstr, (unsigned long)chunk.sizeleft, chunk.buf); + ; + } + + curl_slist_free_all(list); /* free the list again */ + + /* cleanup curl stuff */ + //curl_easy_cleanup(curl_handle); + + free(chunk.buf); + + /* we're done with libcurl, so clean it up */ + //curl_global_cleanup(); + + return 0; +} +#endif + +char* getTagValueFromTagSample( SSuperTable* stbInfo, int tagUsePos) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos); + + return dataBuf; +} + +char* generateTagVaulesForStb(SSuperTable* stbInfo) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "("); + for (int i = 0; i < stbInfo->tagCount; i++) { + if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", 5))) { + if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) { + printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); + tmfree(dataBuf); + return NULL; + } + + char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); + tmfree(dataBuf); + return NULL; + } + rand_string(buf, stbInfo->tags[i].dataLen); + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", 3)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", 6)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", 5)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", 6)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", 8)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", 7)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", 4)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", 4)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); + } else { + printf("No support data type: %s\n", stbInfo->tags[i].dataType); + tmfree(dataBuf); + return NULL; + } + } + dataLen -= 2; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); + return dataBuf; +} + +static int calcRowLen(SSuperTable* superTbls) { + int colIndex; + int lenOfOneRow = 0; + + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + lenOfOneRow += 21; + } else { + printf("get error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + + int tagIndex; + int lenOfTagOfOneRow = 0; + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + printf("get error tag type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + return 0; +} + + +static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, int* childTblCountOfSuperTbl) { + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + char* childTblName = *childTblNameOfSuperTbl; + + //get all child table name use cmd: select tbname from superTblName; + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s", dbName, sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + + int childTblCount = 10000; + count = 0; + childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); + char* pTblName = childTblName; + while ((row = taos_fetch_row(res)) != NULL) { + strncpy(pTblName, (char *)row[0], TSDB_TABLE_NAME_LEN); + //printf("==== sub table name: %s\n", pTblName); + count++; + if (count == childTblCount) { + char *tmp = realloc(childTblName, (size_t)count*1.5*TSDB_TABLE_NAME_LEN); + if (tmp != NULL) { + childTblName = tmp; + memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, (size_t)(count*0.5*TSDB_TABLE_NAME_LEN)); + } else { + // exit, if allocate more memory failed + printf("realloc fail for save child table name of %s.%s\n", dbName, sTblName); + tmfree(childTblName); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + } + pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; + } + + *childTblCountOfSuperTbl = count; + *childTblNameOfSuperTbl = childTblName; + + taos_free_result(res); + return 0; +} + +static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + //get schema use cmd: describe superTblName; + snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + return -1; + } + + int tagIndex = 0; + int columnIndex = 0; + TAOS_FIELD *fields = taos_fetch_fields(res); + while ((row = taos_fetch_row(res)) != NULL) { + if (0 == count) { + count++; + continue; + } + + if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { + strncpy(superTbls->tags[tagIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + strncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + strncpy(superTbls->tags[tagIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + tagIndex++; + } else { + strncpy(superTbls->columns[columnIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + strncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + strncpy(superTbls->columns[columnIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + columnIndex++; + } + count++; + } + + superTbls->columnCount = columnIndex; + superTbls->tagCount = tagIndex; + taos_free_result(res); + + calcRowLen(superTbls); + + if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { + //get all child table name use cmd: select tbname from superTblName; + getAllChildNameOfSuperTable(taos, dbName, superTbls->sTblName, &superTbls->childTblName, &superTbls->childTblCount); + } + return 0; +} + +static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { + char command[BUFFER_SIZE] = "\0"; + + char cols[STRING_LEN] = "\0"; + int colIndex; + int len = 0; + + int lenOfOneRow = 0; + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "BINARY", superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "NCHAR", superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT"); + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT"); + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE"); + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP"); + lenOfOneRow += 21; + } else { + taos_close(taos); + printf("config error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); + + // save for creating child table + superTbls->colsOfCreatChildTable = (char*)calloc(len+20, 1); + if (NULL == superTbls->colsOfCreatChildTable) { + printf("Failed when calloc, size:%d", len+1); + taos_close(taos); + exit(-1); + } + snprintf(superTbls->colsOfCreatChildTable, len+20, "(ts timestamp%s)", cols); + + if (use_metric) { + char tags[STRING_LEN] = "\0"; + int tagIndex; + len = 0; + + int lenOfTagOfOneRow = 0; + len += snprintf(tags + len, STRING_LEN - len, "("); + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "BINARY", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "NCHAR", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "INT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BIGINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "SMALLINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "TINYINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BOOL"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "FLOAT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "DOUBLE"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + taos_close(taos); + printf("config error tag type : %s\n", dataType); + exit(-1); + } + } + len -= 2; + len += snprintf(tags + len, STRING_LEN - len, ")"); + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s (ts timestamp%s) tags %s", dbName, superTbls->sTblName, cols, tags); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + return -1; + } + printf("\ncreate supertable %s success!\n\n", superTbls->sTblName); + } + return 0; +} + + +static int createDatabases() { + TAOS * taos = NULL; + int ret = 0; + taos_init(); + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + char command[BUFFER_SIZE] = "\0"; + + + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].drop) { + sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + taos_close(taos); + return -1; + } + } + + int dataLen = 0; + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "create database if not exists %s ", g_Dbs.db[i].dbName); + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "blocks %d ", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "cache %d ", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "days %d ", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "keep %d ", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "replica %d ", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "update %d ", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "minrows %d ", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "maxrows %d ", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "comp %d ", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "wal %d ", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "fsync %d ", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } + + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + taos_close(taos); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); + + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // describe super table, if exists + sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS; + ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); + } else { + g_Dbs.db[i].superTbls[j].superTblExists = TBL_ALREADY_EXISTS; + ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j]); + } + + if (0 != ret) { + taos_close(taos); + return -1; + } + } + } + + taos_close(taos); + return 0; +} + + +void * createTable(void *sarg) +{ + char command[BUFFER_SIZE] = "\0"; + + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + int64_t lastPrintTime = taosGetTimestampMs(); + + //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); + for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { + if (0 == g_Dbs.use_metric) { + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable); + } else { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + return NULL; + } + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.%s tags %s;", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + free(tagsValBuf); + } + + if (0 != queryDbExec(winfo->taos, command, NO_INSERT_TYPE)){ + return NULL; + } + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] already create %d - %d tables\n", winfo->threadID, winfo->start_table_id, i); + lastPrintTime = currentPrintTime; + } + } + + return NULL; +} + +void startMultiThreadCreateChildTable(char* cols, int threads, int ntables, char* db_name, SSuperTable* superTblInfo) { + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed\n"); + exit(-1); + } + + if (threads < 1) { + threads = 1; + } + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + b = ntables % threads; + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->superTblInfo = superTblInfo; + t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + last = t_info->end_table_id + 1; + t_info->use_metric = 1; + t_info->cols = cols; + pthread_create(pids + i, NULL, createTable, t_info); + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + taos_close(t_info->taos); + } + + free(pids); + free(infos); +} + + +static void createChildTables() { + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + startMultiThreadCreateChildTable(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, g_Dbs.threadCount, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + } + } +} + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +int readTagFromCsvFileToMem(SSuperTable * supterTblInfo) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + + FILE *fp = fopen(supterTblInfo->tagsFile, "r"); + if (fp == NULL) { + printf("Failed to open tags file: %s, reason:%s\n", supterTblInfo->tagsFile, strerror(errno)); + return -1; + } + + if (supterTblInfo->tagDataBuf) { + free(supterTblInfo->tagDataBuf); + supterTblInfo->tagDataBuf = NULL; + } + + supterTblInfo->tagDataBuf = calloc(supterTblInfo->lenOfTagOfOneRow * MAX_LINE_COUNT_IN_MEM, 1); + if (supterTblInfo->tagDataBuf == NULL) { + printf("Failed to calloc, reason:%s\n", strerror(errno)); + fclose(fp); + return -1; + } + + while ((readLen = getline(&line, &n, fp)) != -1) { + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + memcpy(supterTblInfo->tagDataBuf + supterTblInfo->tagSampleCount * supterTblInfo->lenOfTagOfOneRow, line, readLen); + supterTblInfo->tagSampleCount++; + + if (supterTblInfo->tagSampleCount >= MAX_LINE_COUNT_IN_MEM) { + break; + } + } + + free(line); + fclose(fp); + return 0; +} + +int readSampleFromJsonFileToMem(SSuperTable * supterTblInfo) { + // TODO + return 0; +} + + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sampleBuf) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + int getRows = 0; + + memset(sampleBuf, 0, MAX_SAMPLES_ONCE_FROM_FILE* superTblInfo->lenOfOneRow); + while (1) { + readLen = getline(&line, &n, fp); + if (-1 == readLen) { + if(0 != fseek(fp, 0, SEEK_SET)) { + printf("Failed to fseek file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); + return -1; + } + continue; + } + + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + if (readLen > superTblInfo->lenOfOneRow) { + printf("sample row len[%d] overflow define schema len[%d], so discard this row\n", (int32_t)readLen, superTblInfo->lenOfOneRow); + continue; + } + + memcpy(sampleBuf + getRows * superTblInfo->lenOfOneRow, line, readLen); + getRows++; + + if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { + break; + } + } + + tmfree(line); + return 0; +} + +/* +void readSampleFromFileToMem(SSuperTable * supterTblInfo) { + int ret; + if (0 == strncasecmp(supterTblInfo->sampleFormat, "csv", 3)) { + ret = readSampleFromCsvFileToMem(supterTblInfo); + } else if (0 == strncasecmp(supterTblInfo->sampleFormat, "json", 4)) { + ret = readSampleFromJsonFileToMem(supterTblInfo); + } + + if (0 != ret) { + exit(-1); + } +} +*/ +static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* superTbls) { + bool ret = false; + + // columns + cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); + if (columns && columns->type != cJSON_Array) { + printf("failed to read json, columns not found\n"); + goto PARSE_OVER; + } else if (NULL == columns) { + superTbls->columnCount = 0; + superTbls->tagCount = 0; + return true; + } + + int columnSize = cJSON_GetArraySize(columns); + if (columnSize > MAX_COLUMN_COUNT) { + printf("failed to read json, column size overflow, max column size is %d\n", MAX_COLUMN_COUNT); + goto PARSE_OVER; + } + + int count = 1; + int index = 0; + StrColumn columnCase; + + //superTbls->columnCount = columnSize; + for (int k = 0; k < columnSize; ++k) { + cJSON* column = cJSON_GetArrayItem(columns, k); + if (column == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(column, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("failed to read json, column count not found"); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(column, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + printf("failed to read json, column type not found"); + goto PARSE_OVER; + } + //strncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + strncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(column, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + printf("failed to read json, column len not found"); + goto PARSE_OVER; + } else { + columnCase.dataLen = 8; + } + + for (int n = 0; n < count; ++n) { + strncpy(superTbls->columns[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->columns[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->columnCount = index; + + count = 1; + index = 0; + // tags + cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); + if (!tags || tags->type != cJSON_Array) { + printf("failed to read json, tags not found"); + goto PARSE_OVER; + } + + int tagSize = cJSON_GetArraySize(tags); + if (tagSize > MAX_TAG_COUNT) { + printf("failed to read json, tags size overflow, max tag size is %d\n", MAX_TAG_COUNT); + goto PARSE_OVER; + } + + //superTbls->tagCount = tagSize; + for (int k = 0; k < tagSize; ++k) { + cJSON* tag = cJSON_GetArrayItem(tags, k); + if (tag == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(tag, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("failed to read json, column count not found"); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(tag, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + printf("failed to read json, tag type not found"); + goto PARSE_OVER; + } + strncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + printf("failed to read json, column len not found"); + goto PARSE_OVER; + } else { + columnCase.dataLen = 0; + } + + for (int n = 0; n < count; ++n) { + strncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->tags[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->tagCount = index; + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromInsertJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + strncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + strncpy(g_Dbs.host, host->valuestring, MAX_DB_NAME_SIZE); + } else if (!host) { + strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_Dbs.port = port->valueint; + } else if (!port) { + g_Dbs.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + strncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE); + } else if (!user) { + strncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE); + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + strncpy(g_Dbs.password, password->valuestring, MAX_DB_NAME_SIZE); + } else if (!password) { + strncpy(g_Dbs.password, "taosdata", MAX_DB_NAME_SIZE); + } + + cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); + if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { + strncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); + } else if (!resultfile) { + strncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); + } + + cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); + if (threads && threads->type == cJSON_Number) { + g_Dbs.threadCount = threads->valueint; + } else if (!threads) { + g_Dbs.threadCount = 1; + } else { + printf("failed to read json, threads not found"); + goto PARSE_OVER; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (!dbs || dbs->type != cJSON_Array) { + printf("failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + int dbSize = cJSON_GetArraySize(dbs); + if (dbSize > MAX_DB_COUNT) { + printf("failed to read json, databases size overflow, max database is %d\n", MAX_DB_COUNT); + goto PARSE_OVER; + } + + g_Dbs.dbCount = dbSize; + for (int i = 0; i < dbSize; ++i) { + cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); + if (dbinfos == NULL) continue; + + // dbinfo + cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); + if (!dbinfo || dbinfo->type != cJSON_Object) { + printf("failed to read json, dbinfo not found"); + goto PARSE_OVER; + } + + cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); + if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { + printf("failed to read json, db name not found"); + goto PARSE_OVER; + } + strncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); + + cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); + if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { + if (0 == strncasecmp(drop->valuestring, "yes", 3)) { + g_Dbs.db[i].drop = 1; + } else { + g_Dbs.db[i].drop = 0; + } + } else if (!drop) { + g_Dbs.db[i].drop = 0; + } else { + printf("failed to read json, drop not found"); + goto PARSE_OVER; + } + + cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); + if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { + strncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, MAX_DB_NAME_SIZE); + } else if (!precision) { + //strncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, precision not found"); + goto PARSE_OVER; + } + + cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); + if (update && update->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.update = update->valueint; + } else if (!update) { + g_Dbs.db[i].dbCfg.update = -1; + } else { + printf("failed to read json, update not found"); + goto PARSE_OVER; + } + + cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); + if (replica && replica->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.replica = replica->valueint; + } else if (!replica) { + g_Dbs.db[i].dbCfg.replica = -1; + } else { + printf("failed to read json, replica not found"); + goto PARSE_OVER; + } + + cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); + if (keep && keep->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.keep = keep->valueint; + } else if (!keep) { + g_Dbs.db[i].dbCfg.keep = -1; + } else { + printf("failed to read json, keep not found"); + goto PARSE_OVER; + } + + cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); + if (days && days->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.days = days->valueint; + } else if (!days) { + g_Dbs.db[i].dbCfg.days = -1; + } else { + printf("failed to read json, days not found"); + goto PARSE_OVER; + } + + cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); + if (cache && cache->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cache = cache->valueint; + } else if (!cache) { + g_Dbs.db[i].dbCfg.cache = -1; + } else { + printf("failed to read json, cache not found"); + goto PARSE_OVER; + } + + cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); + if (blocks && blocks->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.blocks = blocks->valueint; + } else if (!blocks) { + g_Dbs.db[i].dbCfg.blocks = -1; + } else { + printf("failed to read json, block not found"); + goto PARSE_OVER; + } + + //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); + //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; + //} else if (!maxtablesPerVnode) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; + //} else { + // printf("failed to read json, maxtablesPerVnode not found"); + // goto PARSE_OVER; + //} + + cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); + if (minRows && minRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.minRows = minRows->valueint; + } else if (!minRows) { + g_Dbs.db[i].dbCfg.minRows = -1; + } else { + printf("failed to read json, minRows not found"); + goto PARSE_OVER; + } + + cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); + if (maxRows && maxRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; + } else if (!maxRows) { + g_Dbs.db[i].dbCfg.maxRows = -1; + } else { + printf("failed to read json, maxRows not found"); + goto PARSE_OVER; + } + + cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); + if (comp && comp->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.comp = comp->valueint; + } else if (!comp) { + g_Dbs.db[i].dbCfg.comp = -1; + } else { + printf("failed to read json, comp not found"); + goto PARSE_OVER; + } + + cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); + if (walLevel && walLevel->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; + } else if (!walLevel) { + g_Dbs.db[i].dbCfg.walLevel = -1; + } else { + printf("failed to read json, walLevel not found"); + goto PARSE_OVER; + } + + cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); + if (quorum && quorum->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.quorum = quorum->valueint; + } else if (!quorum) { + g_Dbs.db[i].dbCfg.quorum = -1; + } else { + printf("failed to read json, walLevel not found"); + goto PARSE_OVER; + } + + cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); + if (fsync && fsync->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.fsync = fsync->valueint; + } else if (!fsync) { + g_Dbs.db[i].dbCfg.fsync = -1; + } else { + printf("failed to read json, fsync not found"); + goto PARSE_OVER; + } + + // super_talbes + cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); + if (!stables || stables->type != cJSON_Array) { + printf("failed to read json, super_tables not found"); + goto PARSE_OVER; + } + + int stbSize = cJSON_GetArraySize(stables); + if (stbSize > MAX_SUPER_TABLE_COUNT) { + printf("failed to read json, databases size overflow, max database is %d\n", MAX_SUPER_TABLE_COUNT); + goto PARSE_OVER; + } + + g_Dbs.db[i].superTblCount = stbSize; + for (int j = 0; j < stbSize; ++j) { + cJSON* stbInfo = cJSON_GetArrayItem(stables, j); + if (stbInfo == NULL) continue; + + // dbinfo + cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); + if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { + printf("failed to read json, stb name not found"); + goto PARSE_OVER; + } + strncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); + + cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); + if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { + printf("failed to read json, childtable_prefix not found"); + goto PARSE_OVER; + } + strncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); + + cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null + if (autoCreateTbl && autoCreateTbl->type == cJSON_String && autoCreateTbl->valuestring != NULL) { + if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; + } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + } else if (!autoCreateTbl) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + printf("failed to read json, auto_create_table not found"); + goto PARSE_OVER; + } + + cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no + if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) { + if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; + } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } + } else if (!childTblExists) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + printf("failed to read json, child_table_exists not found"); + goto PARSE_OVER; + } + + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); + if (!count || count->type != cJSON_Number || 0 >= count->valueint) { + printf("failed to read json, childtable_count not found"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; + + cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); + if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].dataSource, dataSource->valuestring, MAX_DB_NAME_SIZE); + } else if (!dataSource) { + strncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, data_source not found"); + goto PARSE_OVER; + } + + cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful + if (insertMode && insertMode->type == cJSON_String && insertMode->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].insertMode, insertMode->valuestring, MAX_DB_NAME_SIZE); + #ifndef TD_LOWA_CURL + if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 7)) { + printf("There no libcurl, so no support resetful test! please use taosc mode.\n"); + goto PARSE_OVER; + } + #endif + } else if (!insertMode) { + strncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, insert_mode not found"); + goto PARSE_OVER; + } + + cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); + if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].startTimestamp, ts->valuestring, MAX_DB_NAME_SIZE); + } else if (!ts) { + strncpy(g_Dbs.db[i].superTbls[j].startTimestamp, "now", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, start_timestamp not found"); + goto PARSE_OVER; + } + + cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); + if (timestampStep && timestampStep->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; + } else if (!timestampStep) { + g_Dbs.db[i].superTbls[j].timeStampStep = 1000; + } else { + printf("failed to read json, timestamp_step not found"); + goto PARSE_OVER; + } + + cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size"); + if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint; + if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } + } else if (!sampleDataBufSize) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } else { + printf("failed to read json, sample_buf_size not found"); + goto PARSE_OVER; + } + + cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); + if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].sampleFormat, sampleFormat->valuestring, MAX_DB_NAME_SIZE); + } else if (!sampleFormat) { + strncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, sample_format not found"); + goto PARSE_OVER; + } + + cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); + if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].sampleFile, sampleFile->valuestring, MAX_FILE_NAME_LEN); + } else if (!sampleFile) { + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, sample_file not found"); + goto PARSE_OVER; + } + + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); + if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].tagsFile, tagsFile->valuestring, MAX_FILE_NAME_LEN); + if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + g_Dbs.db[i].superTbls[j].tagSource = 1; + } + } else if (!tagsFile) { + memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + printf("failed to read json, tags_file not found"); + goto PARSE_OVER; + } + + cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + int32_t len = maxSqlLen->valueint; + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < TSDB_MAX_SQL_LEN) { + len = TSDB_MAX_SQL_LEN; + } + g_Dbs.db[i].superTbls[j].maxSqlLen = len; + } else if (!maxSqlLen) { + g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; + } else { + printf("failed to read json, maxSqlLen not found"); + goto PARSE_OVER; + } + + cJSON *multiThreadWriteOneTbl = cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes + if (multiThreadWriteOneTbl && multiThreadWriteOneTbl->type == cJSON_String && multiThreadWriteOneTbl->valuestring != NULL) { + if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; + } else { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } + } else if (!multiThreadWriteOneTbl) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } else { + printf("failed to read json, multiThreadWriteOneTbl not found"); + goto PARSE_OVER; + } + + cJSON* numberOfTblInOneSql = cJSON_GetObjectItem(stbInfo, "number_of_tbl_in_one_sql"); + if (numberOfTblInOneSql && numberOfTblInOneSql->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = numberOfTblInOneSql->valueint; + } else if (!numberOfTblInOneSql) { + g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = 0; + } else { + printf("failed to read json, numberOfTblInOneSql not found"); + goto PARSE_OVER; + } + + cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "rows_per_tbl"); + if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint; + } else if (!rowsPerTbl) { + g_Dbs.db[i].superTbls[j].rowsPerTbl = 1; + } else { + printf("failed to read json, rowsPerTbl not found"); + goto PARSE_OVER; + } + + cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); + if (disorderRatio && disorderRatio->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; + } else if (!disorderRatio) { + g_Dbs.db[i].superTbls[j].disorderRatio = 0; + } else { + printf("failed to read json, disorderRatio not found"); + goto PARSE_OVER; + } + + cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); + if (disorderRange && disorderRange->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; + } else if (!disorderRange) { + g_Dbs.db[i].superTbls[j].disorderRange = 1000; + } else { + printf("failed to read json, disorderRange not found"); + goto PARSE_OVER; + } + + cJSON* insertRate = cJSON_GetObjectItem(stbInfo, "insert_rate"); + if (insertRate && insertRate->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertRate = insertRate->valueint; + } else if (!insertRate) { + g_Dbs.db[i].superTbls[j].insertRate = 0; + } else { + printf("failed to read json, insert_rate not found"); + goto PARSE_OVER; + } + + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); + if (insertRows && insertRows->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; + if (0 == g_Dbs.db[i].superTbls[j].insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } + } else if (!insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } else { + printf("failed to read json, insert_rows not found"); + goto PARSE_OVER; + } + + if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + + int retVal = getColumnAndTagTypeFromInsertJsonFile(stbInfo, &g_Dbs.db[i].superTbls[j]); + if (false == retVal) { + goto PARSE_OVER; + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromQueryJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + strncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + strncpy(g_queryInfo.host, host->valuestring, MAX_DB_NAME_SIZE); + } else if (!host) { + strncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_queryInfo.port = port->valueint; + } else if (!port) { + g_queryInfo.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + strncpy(g_queryInfo.user, user->valuestring, MAX_DB_NAME_SIZE); + } else if (!user) { + strncpy(g_queryInfo.user, "root", MAX_DB_NAME_SIZE); ; + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + strncpy(g_queryInfo.password, password->valuestring, MAX_DB_NAME_SIZE); + } else if (!password) { + strncpy(g_queryInfo.password, "taosdata", MAX_DB_NAME_SIZE);; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { + strncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); + } else if (!dbs) { + printf("failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); + if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { + strncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); + } else if (!queryMode) { + strncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); + } else { + printf("failed to read json, query_mode not found\n"); + goto PARSE_OVER; + } + + // super_table_query + cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query"); + if (!superQuery) { + g_queryInfo.superQueryInfo.concurrent = 0; + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superQuery->type != cJSON_Object) { + printf("failed to read json, super_table_query not found"); + goto PARSE_OVER; + } else { + cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval"); + if (rate && rate->type == cJSON_Number) { + g_queryInfo.superQueryInfo.rate = rate->valueint; + } else if (!rate) { + g_queryInfo.superQueryInfo.rate = 0; + } + + cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent"); + if (concurrent && concurrent->type == cJSON_Number) { + g_queryInfo.superQueryInfo.concurrent = concurrent->valueint; + } else if (!concurrent) { + g_queryInfo.superQueryInfo.concurrent = 1; + } + + cJSON* mode = cJSON_GetObjectItem(superQuery, "mode"); + if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { + if (0 == strcmp("sync", mode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", mode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 1; + } else { + printf("failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } + + cJSON* interval = cJSON_GetObjectItem(superQuery, "interval"); + if (interval && interval->type == cJSON_Number) { + g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint; + } else if (!interval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.subscribeInterval = 10000; + } + + cJSON* restart = cJSON_GetObjectItem(superQuery, "restart"); + if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { + if (0 == strcmp("yes", restart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", restart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 0; + } else { + printf("failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } + + cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (keepProgress && keepProgress->type == cJSON_String && keepProgress->valuestring != NULL) { + if (0 == strcmp("yes", keepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", keepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } else { + printf("failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!superSqls) { + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superSqls->type != cJSON_Array) { + printf("failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(superSqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.superQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(superSqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("failed to read json, sql not found\n"); + goto PARSE_OVER; + } + strncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { + strncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, super query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + // sub_table_query + cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query"); + if (!subQuery) { + g_queryInfo.subQueryInfo.threadCnt = 0; + g_queryInfo.subQueryInfo.sqlCount = 0; + } else if (subQuery->type != cJSON_Object) { + printf("failed to read json, sub_table_query not found"); + ret = true; + goto PARSE_OVER; + } else { + cJSON* subrate = cJSON_GetObjectItem(subQuery, "query_interval"); + if (subrate && subrate->type == cJSON_Number) { + g_queryInfo.subQueryInfo.rate = subrate->valueint; + } else if (!subrate) { + g_queryInfo.subQueryInfo.rate = 0; + } + + cJSON* threads = cJSON_GetObjectItem(subQuery, "threads"); + if (threads && threads->type == cJSON_Number) { + g_queryInfo.subQueryInfo.threadCnt = threads->valueint; + } else if (!threads) { + g_queryInfo.subQueryInfo.threadCnt = 1; + } + + //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count"); + //if (subTblCnt && subTblCnt->type == cJSON_Number) { + // g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint; + //} else if (!subTblCnt) { + // g_queryInfo.subQueryInfo.childTblCount = 0; + //} + + cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname"); + if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { + strncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); + } else { + printf("failed to read json, super table name not found\n"); + goto PARSE_OVER; + } + + cJSON* submode = cJSON_GetObjectItem(subQuery, "mode"); + if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { + if (0 == strcmp("sync", submode->valuestring)) { + g_queryInfo.subQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", submode->valuestring)) { + g_queryInfo.subQueryInfo.subscribeMode = 1; + } else { + printf("failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeMode = 0; + } + + cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval"); + if (subinterval && subinterval->type == cJSON_Number) { + g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint; + } else if (!subinterval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.subQueryInfo.subscribeInterval = 10000; + } + + cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart"); + if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { + if (0 == strcmp("yes", subrestart->valuestring)) { + g_queryInfo.subQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", subrestart->valuestring)) { + g_queryInfo.subQueryInfo.subscribeRestart = 0; + } else { + printf("failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeRestart = 1; + } + + cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress"); + if (subkeepProgress && subkeepProgress->type == cJSON_String && subkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", subkeepProgress->valuestring)) { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + } else { + printf("failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls"); + if (!subsqls) { + g_queryInfo.subQueryInfo.sqlCount = 0; + } else if (subsqls->type != cJSON_Array) { + printf("failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(subsqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.subQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(subsqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("failed to read json, sql not found\n"); + goto PARSE_OVER; + } + strncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ + strncpy(g_queryInfo.subQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, sub query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getInfoFromJsonFile(char* file) { + FILE *fp = fopen(file, "r"); + if (!fp) { + printf("failed to read %s, reason:%s\n", file, strerror(errno)); + return false; + } + + bool ret = false; + int maxLen = 64000; + char *content = calloc(1, maxLen + 1); + int len = fread(content, 1, maxLen, fp); + if (len <= 0) { + free(content); + fclose(fp); + printf("failed to read %s, content is null", file); + return false; + } + + content[len] = 0; + cJSON* root = cJSON_Parse(content); + if (root == NULL) { + printf("failed to cjson parse %s, invalid json format", file); + goto PARSE_OVER; + } + + cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); + if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { + if (0 == strcasecmp("insert", filetype->valuestring)) { + g_jsonType = INSERT_MODE; + } else if (0 == strcasecmp("query", filetype->valuestring)) { + g_jsonType = QUERY_MODE; + } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { + g_jsonType = SUBSCRIBE_MODE; + } else { + printf("failed to read json, filetype not support\n"); + goto PARSE_OVER; + } + } else if (!filetype) { + g_jsonType = INSERT_MODE; + } else { + printf("failed to read json, filetype not found\n"); + goto PARSE_OVER; + } + + if (INSERT_MODE == g_jsonType) { + ret = getMetaFromInsertJsonFile(root); + } else if (QUERY_MODE == g_jsonType) { + ret = getMetaFromQueryJsonFile(root); + } else if (SUBSCRIBE_MODE == g_jsonType) { + ret = getMetaFromQueryJsonFile(root); + } else { + printf("input json file type error! please input correct file type: insert or query or subscribe\n"); + goto PARSE_OVER; + } + +PARSE_OVER: + free(content); + cJSON_Delete(root); + fclose(fp); + return ret; +} + + +void prePareSampleData() { + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + //if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].dataSource, "sample", 6)) { + // readSampleFromFileToMem(&g_Dbs.db[i].superTbls[j]); + //} + + if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { + (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]); + } + + #ifdef TD_LOWA_CURL + if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 8)) { + curl_global_init(CURL_GLOBAL_ALL); + } + #endif + } + } +} + +void postFreeResource() { + tmfclose(g_fpOfInsertResult); + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (0 != g_Dbs.db[i].superTbls[j].colsOfCreatChildTable) { + free(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable); + g_Dbs.db[i].superTbls[j].colsOfCreatChildTable = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { + free(g_Dbs.db[i].superTbls[j].sampleDataBuf); + g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].childTblName) { + free(g_Dbs.db[i].superTbls[j].childTblName); + g_Dbs.db[i].superTbls[j].childTblName = NULL; + } + + #ifdef TD_LOWA_CURL + if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 8)) { + curl_global_cleanup(); + } + #endif + } + } +} + +int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* superTblInfo, int* sampleUsePos, FILE *fp, char* sampleBuf) { + if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) { + int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleBuf); + if (0 != ret) { + return -1; + } + *sampleUsePos = 0; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%s", sampleBuf + superTblInfo->lenOfOneRow * (*sampleUsePos)); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + + (*sampleUsePos)++; + + return dataLen; +} + +int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) { + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + for (int i = 0; i < stbInfo->columnCount; i++) { + if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { + if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { + printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); + return (-1); + } + + char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", stbInfo->columns[i].dataLen); + return (-1); + } + rand_string(buf, stbInfo->columns[i].dataLen); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "int", 3)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bigint", 6)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "float", 5)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "double", 6)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "smallint", 8)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + } else { + printf("No support data type: %s\n", stbInfo->columns[i].dataType); + return (-1); + } + } + dataLen -= 2; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + + return dataLen; +} + +void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDataBuf) { + SSuperTable* superTblInfo = winfo->superTblInfo; + + int samplePos = 0; + + //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); + int64_t totalRowsInserted = 0; + int64_t totalAffectedRows = 0; + int64_t lastPrintTime = taosGetTimestampMs(); + + char* buffer = calloc(superTblInfo->maxSqlLen+1, 1); + if (NULL == buffer) { + printf("========calloc size[ %d ] fail!\n", superTblInfo->maxSqlLen); + return; + } + + int32_t numberOfTblInOneSql = superTblInfo->numberOfTblInOneSql; + int32_t tbls = winfo->end_table_id - winfo->start_table_id + 1; + if (numberOfTblInOneSql > tbls) { + numberOfTblInOneSql = tbls; + } + + int64_t time_counter = winfo->start_time; + int64_t tmp_time; + int sampleUsePos; + + int64_t st = 0; + int64_t et = 0; + for (int i = 0; i < superTblInfo->insertRows;) { + if (superTblInfo->insertRate && (et - st) < 1000) { + taosMsleep(1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + if (superTblInfo->insertRate) { + st = taosGetTimestampMs(); + } + + int32_t tbl_id = 0; + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { + int inserted = i; + + int k = 0; + int batchRowsSql = 0; + while (1) + { + int len = 0; + memset(buffer, 0, superTblInfo->maxSqlLen); + char *pstr = buffer; + + int32_t end_tbl_id = tID + numberOfTblInOneSql; + if (end_tbl_id > winfo->end_table_id) { + end_tbl_id = winfo->end_table_id+1; + } + for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) { + sampleUsePos = samplePos; + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample(superTblInfo, tbl_id % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + goto free_and_statistics; + } + + if (0 == len) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + } + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + if (0 == len) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + } + } else { // pre-create child table + if (0 == len) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); + } + } + + tmp_time = time_counter; + for (k = 0; k < superTblInfo->rowsPerTbl;) { + int retLen = 0; + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); + if (retLen < 0) { + goto free_and_statistics; + } + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { + int64_t d = tmp_time - rand() % superTblInfo->disorderRange; + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); + } else { + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); + } + if (retLen < 0) { + goto free_and_statistics; + } + } + len += retLen; + //inserted++; + k++; + totalRowsInserted++; + batchRowsSql++; + + if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128) || batchRowsSql >= INT16_MAX - 1) { + tID = tbl_id + 1; + printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", superTblInfo->lenOfOneRow); + goto send_to_server; + } + } + + } + + tID = tbl_id; + inserted += superTblInfo->rowsPerTbl; + + send_to_server: + batchRowsSql = 0; + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + //printf("multi table===== sql: %s \n\n", buffer); + //int64_t t1 = taosGetTimestampMs(); + int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + if (0 > affectedRows) { + goto free_and_statistics; + } + totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + lastPrintTime = currentPrintTime; + } + //int64_t t2 = taosGetTimestampMs(); + //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); + } else { + #ifdef TD_LOWA_CURL + //int64_t t1 = taosGetTimestampMs(); + int retCode = curlProceSql(g_Dbs.host, g_Dbs.port, buffer, winfo->curl_handle); + //int64_t t2 = taosGetTimestampMs(); + //printf("http insert sql return, Spent %ld ms \n", t2 - t1); + + if (0 != retCode) { + printf("========curl return fail, threadID[%d]\n", winfo->threadID); + goto free_and_statistics; + } + #else + printf("========no use http mode for no curl lib!\n"); + goto free_and_statistics; + #endif + } + + //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); + break; + } + + if (tID > winfo->end_table_id) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + samplePos = sampleUsePos; + } + i = inserted; + time_counter = tmp_time; + } + } + + if (superTblInfo->insertRate) { + et = taosGetTimestampMs(); + } + //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); + } + + free_and_statistics: + tmfree(buffer); + winfo->totalRowsInserted = totalRowsInserted; + winfo->totalAffectedRows = totalAffectedRows; + printf("====thread[%d] completed total inserted rows: %"PRId64 ", affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + return; +} + +// sync insertion +/* + 1 thread: 100 tables * 2000 rows/s + 1 thread: 10 tables * 20000 rows/s + 6 thread: 300 tables * 2000 rows/s + + 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s +*/ +void *syncWrite(void *sarg) { + int64_t totalRowsInserted = 0; + int64_t totalAffectedRows = 0; + int64_t lastPrintTime = taosGetTimestampMs(); + + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + FILE *fp = NULL; + char* sampleDataBuf = NULL; + int samplePos = 0; + + // each thread read sample data from csv file + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + sampleDataBuf = calloc(superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); + if (sampleDataBuf == NULL) { + printf("Failed to calloc %d Bytes, reason:%s\n", superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); + return NULL; + } + + fp = fopen(superTblInfo->sampleFile, "r"); + if (fp == NULL) { + printf("Failed to open sample file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); + tmfree(sampleDataBuf); + return NULL; + } + int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleDataBuf); + if (0 != ret) { + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; + } + } + + if (superTblInfo->numberOfTblInOneSql > 0) { + syncWriteForNumberOfTblInOneSql(winfo, fp, sampleDataBuf); + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; + } + + //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); + + char* buffer = calloc(superTblInfo->maxSqlLen, 1); + + int nrecords_per_request = 0; + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + nrecords_per_request = (superTblInfo->maxSqlLen - 1280 - superTblInfo->lenOfTagOfOneRow) / superTblInfo->lenOfOneRow; + } else { + nrecords_per_request = (superTblInfo->maxSqlLen - 1280) / superTblInfo->lenOfOneRow; + } + + int nrecords_no_last_req = nrecords_per_request; + int nrecords_last_req = 0; + int loop_cnt = 0; + if (0 != superTblInfo->insertRate) { + if (nrecords_no_last_req >= superTblInfo->insertRate) { + nrecords_no_last_req = superTblInfo->insertRate; + } else { + nrecords_last_req = superTblInfo->insertRate % nrecords_per_request; + loop_cnt = (superTblInfo->insertRate / nrecords_per_request) + (superTblInfo->insertRate % nrecords_per_request ? 1 : 0) ; + } + } + + if (nrecords_no_last_req <= 0) { + nrecords_no_last_req = 1; + } + + if (nrecords_no_last_req >= INT16_MAX) { + nrecords_no_last_req = INT16_MAX - 1; + } + + if (nrecords_last_req >= INT16_MAX) { + nrecords_last_req = INT16_MAX - 1; + } + + int nrecords_cur_req = nrecords_no_last_req; + int loop_cnt_orig = loop_cnt; + + //printf("========nrecords_per_request:%d, nrecords_no_last_req:%d, nrecords_last_req:%d, loop_cnt:%d\n", nrecords_per_request, nrecords_no_last_req, nrecords_last_req, loop_cnt); + + int64_t time_counter = winfo->start_time; + + int64_t st = 0; + int64_t et = 0; + for (int i = 0; i < superTblInfo->insertRows;) { + if (superTblInfo->insertRate && (et - st) < 1000) { + taosMsleep(1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + if (superTblInfo->insertRate) { + st = taosGetTimestampMs(); + } + + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { + int inserted = i; + int64_t tmp_time = time_counter; + + int sampleUsePos = samplePos; + int k = 0; + while (1) + { + int len = 0; + memset(buffer, 0, superTblInfo->maxSqlLen); + char *pstr = buffer; + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample(superTblInfo, tID % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + goto free_and_statistics_2; + } + + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values", winfo->db_name, superTblInfo->childTblPrefix, tID, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values", winfo->db_name, superTblInfo->childTblName + tID * TSDB_TABLE_NAME_LEN); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values", winfo->db_name, superTblInfo->childTblPrefix, tID); + } + + for (k = 0; k < nrecords_cur_req;) { + int retLen = 0; + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); + if (retLen < 0) { + goto free_and_statistics_2; + } + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { + int64_t d = tmp_time - rand() % superTblInfo->disorderRange; + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); + //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); + } else { + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); + } + if (retLen < 0) { + goto free_and_statistics_2; + } + } + len += retLen; + inserted++; + k++; + totalRowsInserted++; + + if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) break; + } + + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + //printf("===== sql: %s \n\n", buffer); + //int64_t t1 = taosGetTimestampMs(); + int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + if (0 > affectedRows){ + goto free_and_statistics_2; + } + totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + lastPrintTime = currentPrintTime; + } + //int64_t t2 = taosGetTimestampMs(); + //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); + } else { + #ifdef TD_LOWA_CURL + //int64_t t1 = taosGetTimestampMs(); + int retCode = curlProceSql(g_Dbs.host, g_Dbs.port, buffer, winfo->curl_handle); + //int64_t t2 = taosGetTimestampMs(); + //printf("http insert sql return, Spent %ld ms \n", t2 - t1); + + if (0 != retCode) { + printf("========curl return fail, threadID[%d]\n", winfo->threadID); + goto free_and_statistics_2; + } + #else + printf("========no use http mode for no curl lib!\n"); + goto free_and_statistics_2; + #endif + } + + //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); + + if (loop_cnt) { + loop_cnt--; + if ((1 == loop_cnt) && (0 != nrecords_last_req)) { + nrecords_cur_req = nrecords_last_req; + } else if (0 == loop_cnt){ + nrecords_cur_req = nrecords_no_last_req; + loop_cnt = loop_cnt_orig; + break; + } + } else { + break; + } + } + + if (tID == winfo->end_table_id) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + samplePos = sampleUsePos; + } + i = inserted; + time_counter = tmp_time; + } + } + + if (superTblInfo->insertRate) { + et = taosGetTimestampMs(); + } + //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); + } + + free_and_statistics_2: + tmfree(buffer); + tmfree(sampleDataBuf); + tmfclose(fp); + + winfo->totalRowsInserted = totalRowsInserted; + winfo->totalAffectedRows = totalAffectedRows; + + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + return NULL; +} + +void callBack(void *param, TAOS_RES *res, int code) { + threadInfo* winfo = (threadInfo*)param; + + if (winfo->superTblInfo->insertRate) { + winfo->et = taosGetTimestampMs(); + if (winfo->et - winfo->st < 1000) { + taosMsleep(1000 - (winfo->et - winfo->st)); // ms + } + } + + char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); + char *data = calloc(1, MAX_DATA_SIZE); + char *pstr = buffer; + pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id); + if (winfo->counter >= winfo->superTblInfo->insertRows) { + winfo->start_table_id++; + winfo->counter = 0; + } + if (winfo->start_table_id > winfo->end_table_id) { + tsem_post(&winfo->lock_sem); + free(buffer); + free(data); + taos_free_result(res); + return; + } + + for (int i = 0; i < winfo->nrecords_per_request; i++) { + int rand_num = rand() % 100; + if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) + { + int64_t d = winfo->lastTs - rand() % 1000000 + rand_num; + //generateData(data, datatype, ncols_per_record, d, len_of_binary); + (void)generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo); + } else { + //generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); + (void)generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo); + } + pstr += sprintf(pstr, "%s", data); + winfo->counter++; + + if (winfo->counter >= winfo->superTblInfo->insertRows) { + break; + } + } + + if (winfo->superTblInfo->insertRate) { + winfo->st = taosGetTimestampMs(); + } + taos_query_a(winfo->taos, buffer, callBack, winfo); + free(buffer); + free(data); + + taos_free_result(res); +} + +void *asyncWrite(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + winfo->nrecords_per_request = 0; + //if (AUTO_CREATE_SUBTBL == winfo->superTblInfo->autoCreateTable) { + winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280 - winfo->superTblInfo->lenOfTagOfOneRow) / winfo->superTblInfo->lenOfOneRow; + //} else { + // winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280) / winfo->superTblInfo->lenOfOneRow; + //} + + if (0 != winfo->superTblInfo->insertRate) { + if (winfo->nrecords_per_request >= winfo->superTblInfo->insertRate) { + winfo->nrecords_per_request = winfo->superTblInfo->insertRate; + } + } + + if (winfo->nrecords_per_request <= 0) { + winfo->nrecords_per_request = 1; + } + + if (winfo->nrecords_per_request >= INT16_MAX) { + winfo->nrecords_per_request = INT16_MAX - 1; + } + + if (winfo->nrecords_per_request >= INT16_MAX) { + winfo->nrecords_per_request = INT16_MAX - 1; + } + + winfo->st = 0; + winfo->et = 0; + winfo->lastTs = winfo->start_time; + + if (winfo->superTblInfo->insertRate) { + winfo->st = taosGetTimestampMs(); + } + taos_query_a(winfo->taos, "show databases", callBack, winfo); + + tsem_wait(&(winfo->lock_sem)); + + return NULL; +} + +void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* superTblInfo) { + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); + int ntables = superTblInfo->childTblCount; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + TAOS* taos; + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + if (NULL == taos) { + printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); + exit(-1); + } + } + + int32_t timePrec = TSDB_TIME_PRECISION_MILLI; + if (0 != precision[0]) { + if (0 == strncasecmp(precision, "ms", 2)) { + timePrec = TSDB_TIME_PRECISION_MILLI; + } else if (0 == strncasecmp(precision, "us", 2)) { + timePrec = TSDB_TIME_PRECISION_MICRO; + } else { + printf("No support precision: %s\n", precision); + exit(-1); + } + } + + int64_t start_time; + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + start_time = taosGetTimestamp(timePrec); + } else { + (void)taosParseTime(superTblInfo->startTimestamp, &start_time, strlen(superTblInfo->startTimestamp), timePrec, 0); + } + + double start = getCurrentTime(); + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->superTblInfo = superTblInfo; + + t_info->start_time = start_time; + + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + t_info->taos = taos; + } else { + t_info->taos = NULL; + #ifdef TD_LOWA_CURL + t_info->curl_handle = curl_easy_init(); + #endif + } + + if (0 == superTblInfo->multiThreadWriteOneTbl) { + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + last = t_info->end_table_id + 1; + } else { + t_info->start_table_id = 0; + t_info->end_table_id = superTblInfo->childTblCount - 1; + t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); + } + + tsem_init(&(t_info->lock_sem), 0, 0); + + if (SYNC == g_Dbs.queryMode) { + pthread_create(pids + i, NULL, syncWrite, t_info); + } else { + pthread_create(pids + i, NULL, asyncWrite, t_info); + } + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + + tsem_destroy(&(t_info->lock_sem)); + + superTblInfo->totalAffectedRows += t_info->totalAffectedRows; + superTblInfo->totalRowsInserted += t_info->totalRowsInserted; + #ifdef TD_LOWA_CURL + if (t_info->curl_handle) { + curl_easy_cleanup(t_info->curl_handle); + } + #endif + } + + double end = getCurrentTime(); + + taos_close(taos); + + free(pids); + free(infos); + + printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s\n\n", + end - start, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName); + fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s\n\n", + end - start, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName); +} + + +void *readTable(void *sarg) { +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; + TAOS *taos = rinfo->taos; + char command[BUFFER_SIZE] = "\0"; + int64_t sTime = rinfo->start_time; + char *tb_prefix = rinfo->tb_prefix; + FILE *fp = fopen(rinfo->fp, "a"); + if (NULL == fp) { + printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + return NULL; + } + + int num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; + int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; + int totalData = num_of_DPT * num_of_tables; + bool do_aggreFunc = g_Dbs.do_aggreFunc; + + int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + if (!do_aggreFunc) { + printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); + } + printf("%d records:\n", totalData); + fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); + + for (int j = 0; j < n; j++) { + double totalT = 0; + int count = 0; + for (int i = 0; i < num_of_tables; i++) { + sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime); + + double t = getCurrentTime(); + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + return NULL; + } + + while (taos_fetch_row(pSql) != NULL) { + count++; + } + + t = getCurrentTime() - t; + totalT += t; + + taos_free_result(pSql); + } + + fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", + aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, + (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); + printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT); + } + fprintf(fp, "\n"); + fclose(fp); +#endif + return NULL; +} + +void *readMetric(void *sarg) { +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; + TAOS *taos = rinfo->taos; + char command[BUFFER_SIZE] = "\0"; + FILE *fp = fopen(rinfo->fp, "a"); + if (NULL == fp) { + printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + return NULL; + } + + int num_of_DPT = rinfo->superTblInfo->insertRows; + int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; + int totalData = num_of_DPT * num_of_tables; + bool do_aggreFunc = g_Dbs.do_aggreFunc; + + int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + if (!do_aggreFunc) { + printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); + } + printf("%d records:\n", totalData); + fprintf(fp, "Querying On %d records:\n", totalData); + + for (int j = 0; j < n; j++) { + char condition[BUFFER_SIZE - 30] = "\0"; + char tempS[64] = "\0"; + + int m = 10 < num_of_tables ? 10 : num_of_tables; + + for (int i = 1; i <= m; i++) { + if (i == 1) { + sprintf(tempS, "t1 = %d", i); + } else { + sprintf(tempS, " or t1 = %d ", i); + } + strcat(condition, tempS); + + sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); + + printf("Where condition: %s\n", condition); + fprintf(fp, "%s\n", command); + + double t = getCurrentTime(); + + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + return NULL; + } + int count = 0; + while (taos_fetch_row(pSql) != NULL) { + count++; + } + t = getCurrentTime() - t; + + fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / t, t * 1000); + printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t); + + taos_free_result(pSql); + } + fprintf(fp, "\n"); + } + fclose(fp); +#endif + return NULL; +} + + +int insertTestProcess() { + + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); + if (NULL == g_fpOfInsertResult) { + fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); + return 1; + }; + + printfInsertMeta(); + printfInsertMetaToFile(g_fpOfInsertResult); + + printf("Press enter key to continue\n\n"); + (void)getchar(); + + init_rand_data(); + + // create database and super tables + (void)createDatabases(); + + // pretreatement + prePareSampleData(); + + double start; + double end; + + // create child tables + start = getCurrentTime(); + createChildTables(); + end = getCurrentTime(); + if (g_totalChildTables > 0) { + printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); + fprintf(g_fpOfInsertResult, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); + } + + usleep(1000*1000); + + // create sub threads for inserting data + //start = getCurrentTime(); + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + startMultiThreadInsertData(g_Dbs.threadCount, g_Dbs.db[i].dbName, g_Dbs.db[i].dbCfg.precision, superTblInfo); + } + } + //end = getCurrentTime(); + + //int64_t totalRowsInserted = 0; + //int64_t totalAffectedRows = 0; + //for (int i = 0; i < g_Dbs.dbCount; i++) { + // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // totalRowsInserted += g_Dbs.db[i].superTbls[j].totalRowsInserted; + // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; + //} + //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalRowsInserted, totalAffectedRows, g_Dbs.threadCount); + if (NULL == g_args.metaFile && false == g_Dbs.insert_only) { + // query data + pthread_t read_id; + threadInfo *rInfo = malloc(sizeof(threadInfo)); + rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + rInfo->start_table_id = 0; + rInfo->end_table_id = g_Dbs.db[0].superTbls[0].childTblCount - 1; + //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + //rInfo->nrecords_per_table = g_Dbs.db[0].superTbls[0].insertRows; + rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + rInfo->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); + strcpy(rInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix); + strcpy(rInfo->fp, g_Dbs.resultFile); + + if (!g_Dbs.use_metric) { + pthread_create(&read_id, NULL, readTable, rInfo); + } else { + pthread_create(&read_id, NULL, readMetric, rInfo); + } + pthread_join(read_id, NULL); + taos_close(rInfo->taos); + } + + postFreeResource(); + + return 0; +} + +void *superQueryProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + //char sqlStr[MAX_TB_NAME_SIZE*2]; + //sprintf(sqlStr, "use %s", g_queryInfo.dbName); + //queryDB(winfo->taos, sqlStr); + + int64_t st = 0; + int64_t et = 0; + while (1) { + if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + st = taosGetTimestampMs(); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + int64_t t1 = taosGetTimestampUs(); + selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], g_queryInfo.superQueryInfo.result[i]); + int64_t t2 = taosGetTimestampUs(); + printf("taosc select sql return, Spent %f s\n", (t2 - t1)/1000000.0); + } else { + #ifdef TD_LOWA_CURL + int64_t t1 = taosGetTimestampUs(); + int retCode = curlProceSql(g_queryInfo.host, g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i], winfo->curl_handle); + int64_t t2 = taosGetTimestampUs(); + printf("http select sql return, Spent %f s \n", (t2 - t1)/1000000.0); + + if (0 != retCode) { + printf("========curl return fail, threadID[%d]\n", winfo->threadID); + return NULL; + } + #endif + } + } + et = taosGetTimestampMs(); + printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.6fs\n\n", (uint64_t)pthread_self(), (double)(et - st)/1000.0); + } + return NULL; +} + +void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { + char sourceString[32] = "xxxx"; + char subTblName[MAX_TB_NAME_SIZE*3]; + sprintf(subTblName, "%s.%s", g_queryInfo.dbName, g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); + + //printf("inSql: %s\n", inSql); + + char* pos = strstr(inSql, sourceString); + if (0 == pos) { + return; + } + + strncpy(outSql, inSql, pos - inSql); + //printf("1: %s\n", outSql); + strcat(outSql, subTblName); + //printf("2: %s\n", outSql); + strcat(outSql, pos+strlen(sourceString)); + //printf("3: %s\n", outSql); +} + +void *subQueryProcess(void *sarg) { + char sqlstr[1024]; + threadInfo *winfo = (threadInfo *)sarg; + int64_t st = 0; + int64_t et = 0; + while (1) { + if (g_queryInfo.subQueryInfo.rate && (et - st) < g_queryInfo.subQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.subQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + st = taosGetTimestampMs(); + for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + memset(sqlstr,0,sizeof(sqlstr)); + replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], sqlstr, i); + selectAndGetResult(winfo->taos, sqlstr, g_queryInfo.subQueryInfo.result[i]); + } + } + et = taosGetTimestampMs(); + printf("========thread[%"PRIu64"] complete all sqls to allocate all sub-tables once queries duration:%.4fs\n\n", (uint64_t)pthread_self(), (double)(et - st)/1000.0); + } + return NULL; +} + +int queryTestProcess() { + printfQueryMeta(); + + printf("Press enter key to continue\n\n"); + (void)getchar(); + + TAOS * taos = NULL; + taos_init(); + taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.subQueryInfo.sqlCount) { + (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); + } + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from super table + if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { + + pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + t_info->taos = taos; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + (void)queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE); + } else { + t_info->taos = NULL; + #ifdef TD_LOWA_CURL + t_info->curl_handle = curl_easy_init(); + #endif + } + + pthread_create(pids + i, NULL, superQueryProcess, t_info); + } + }else { + g_queryInfo.superQueryInfo.concurrent = 0; + } + + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + //==== create sub threads for query from sub table + if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + int ntables = g_queryInfo.subQueryInfo.childTblCount; + int threads = g_queryInfo.subQueryInfo.threadCnt; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + t_info->taos = taos; + pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); + } + + g_queryInfo.subQueryInfo.threadCnt = threads; + }else { + g_queryInfo.subQueryInfo.threadCnt = 0; + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); + } + + tmfree((char*)pids); + tmfree((char*)infos); + + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); + + taos_close(taos); + return 0; +} + +static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + printf("failed to subscribe result, code:%d, reason:%s\n", code, taos_errstr(res)); + return; + } + + getResult(res, (char*)param); + taos_free_result(res); +} + +static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { + TAOS_SUB* tsub = NULL; + + if (g_queryInfo.superQueryInfo.subscribeMode) { + tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.superQueryInfo.subscribeInterval); + } else { + tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, NULL, NULL, 0); + } + + if (tsub == NULL) { + printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql); + return NULL; + } + + return tsub; +} + +void *subSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + char subSqlstr[1024]; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){ + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + memset(subSqlstr,0,sizeof(subSqlstr)); + replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i); + g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, g_queryInfo.subQueryInfo.result[i]); + if (NULL == g_queryInfo.subQueryInfo.tsub[i]) { + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", pthread_self(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + while (1) { + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.subQueryInfo.subscribeMode) { + continue; + } + + TAOS_RES* res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]); + if (res) { + getResult(res, g_queryInfo.subQueryInfo.result[i]); + taos_free_result(res); + } + } + } + + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], g_queryInfo.subQueryInfo.subscribeKeepProgress); + } + return NULL; +} + +void *superSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) { + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.superQueryInfo.sql[i], topic, g_queryInfo.superQueryInfo.result[i]); + if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", pthread_self(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + while (1) { + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.superQueryInfo.subscribeMode) { + continue; + } + + TAOS_RES* res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); + if (res) { + getResult(res, g_queryInfo.superQueryInfo.result[i]); + taos_free_result(res); + } + } + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + return NULL; +} + +int subscribeTestProcess() { + printfQueryMeta(); + + printf("Press enter key to continue\n\n"); + (void)getchar(); + + TAOS * taos = NULL; + taos_init(); + taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.subQueryInfo.sqlCount) { + (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); + } + + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from super table + if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { + pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + t_info->taos = taos; + pthread_create(pids + i, NULL, superSubscribeProcess, t_info); + } + } + + //==== create sub threads for query from sub table + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + int ntables = g_queryInfo.subQueryInfo.childTblCount; + int threads = g_queryInfo.subQueryInfo.threadCnt; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + t_info->taos = taos; + pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); + } + g_queryInfo.subQueryInfo.threadCnt = threads; + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); + } + + tmfree((char*)pids); + tmfree((char*)infos); + + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); + taos_close(taos); + return 0; +} + +void initOfInsertMeta() { + memset(&g_Dbs, 0, sizeof(SDbs)); + + // set default values + strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_Dbs.port = 6030; + strncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + strncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); + g_Dbs.threadCount = 2; + g_Dbs.use_metric = true; +} + +void initOfQueryMeta() { + memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); + + // set default values + strncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_queryInfo.port = 6030; + strncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + strncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); +} + +void setParaFromArg(){ + if (g_args.host) { + strcpy(g_Dbs.host, g_args.host); + } else { + strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } + + if (g_args.user) { + strcpy(g_Dbs.user, g_args.user); + } + + if (g_args.password) { + strcpy(g_Dbs.password, g_args.password); + } + + if (g_args.port) { + g_Dbs.port = g_args.port; + } + + g_Dbs.dbCount = 1; + g_Dbs.db[0].drop = 1; + + strncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); + g_Dbs.db[0].dbCfg.replica = g_args.replica; + strncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + + + strncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); + + g_Dbs.use_metric = g_args.use_metric; + g_Dbs.insert_only = g_args.insert_only; + + g_Dbs.db[0].superTblCount = 1; + strncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.queryMode = g_args.mode; + + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].insertRate = 0; + g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; + g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; + strncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + strncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + strncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); + strncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].timeStampStep = 10; + + // g_args.num_of_RPR; + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; + + g_Dbs.do_aggreFunc = true; + + char dataString[STRING_LEN]; + char **data_type = g_args.datatype; + + memset(dataString, 0, STRING_LEN); + + if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 || strcasecmp(data_type[0], "NCHAR") == 0 ) { + g_Dbs.do_aggreFunc = false; + } + + g_Dbs.db[0].superTbls[0].columnCount = 0; + for (int i = 0; i < MAX_NUM_DATATYPE; i++) { + if (data_type[i] == NULL) { + break; + } + + strncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, data_type[i], MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columnCount++; + } + + if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { + g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; + } else { + for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { + strncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount++; + } + } + + if (g_Dbs.use_metric) { + strncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; + + strncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tagCount = 2; + } else { + g_Dbs.db[0].superTbls[0].tagCount = 0; + } +} + +/* Function to do regular expression check */ +static int regexMatch(const char *s, const char *reg, int cflags) { + regex_t regex; + char msgbuf[100] = {0}; + + /* Compile regular expression */ + if (regcomp(®ex, reg, cflags) != 0) { + printf("Fail to compile regex\n"); + exit(-1); + } + + /* Execute regular expression */ + int reti = regexec(®ex, s, 0, NULL, 0); + if (!reti) { + regfree(®ex); + return 1; + } else if (reti == REG_NOMATCH) { + regfree(®ex); + return 0; + } else { + regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); + printf("Regex match failed: %s\n", msgbuf); + regfree(®ex); + exit(-1); + } + + return 0; +} + +static int isCommentLine(char *line) { + if (line == NULL) return 1; + + return regexMatch(line, "^\\s*#.*", REG_EXTENDED); +} + +void querySqlFile(TAOS* taos, char* sqlFile) +{ + FILE *fp = fopen(sqlFile, "r"); + if (fp == NULL) { + printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); + return; + } + + int read_len = 0; + char * cmd = calloc(1, MAX_SQL_SIZE); + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; + + double t = getCurrentTime(); + + while ((read_len = tgetline(&line, &line_len, fp)) != -1) { + if (read_len >= MAX_SQL_SIZE) continue; + line[--read_len] = '\0'; + + if (read_len == 0 || isCommentLine(line)) { // line starts with # + continue; + } + + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } + + memcpy(cmd + cmd_len, line, read_len); + queryDbExec(taos, cmd, NO_INSERT_TYPE); + memset(cmd, 0, MAX_SQL_SIZE); + cmd_len = 0; + } + + t = getCurrentTime() - t; + printf("run %s took %.6f second(s)\n\n", sqlFile, t); + + tmfree(cmd); + tmfree(line); + tmfclose(fp); + return; +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv, &g_args); + + if (g_args.metaFile) { + initOfInsertMeta(); + initOfQueryMeta(); + if (false == getInfoFromJsonFile(g_args.metaFile)) { + printf("Failed to read %s\n", g_args.metaFile); + return 1; + } + } else { + + memset(&g_Dbs, 0, sizeof(SDbs)); + g_jsonType = INSERT_MODE; + setParaFromArg(); + + if (NULL != g_args.sqlFile) { + TAOS* qtaos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); + querySqlFile(qtaos, g_args.sqlFile); + taos_close(qtaos); + return 0; + } + + (void)insertTestProcess(); + if (g_Dbs.insert_only) return 0; + + // select + + //printf("At present, there is no integration of taosdemo, please wait patiently!\n"); + return 0; + } + + if (INSERT_MODE == g_jsonType) { + if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); + (void)insertTestProcess(); + } else if (QUERY_MODE == g_jsonType) { + if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + (void)queryTestProcess(); + } else if (SUBSCRIBE_MODE == g_jsonType) { + if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + (void)subscribeTestProcess(); + } else { + ; + } + + taos_cleanup(); + return 0; +} + diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index bdfea26294201e4ee8a96acc3ec0a8c17c524f13..60707f22e297e3cd253d88cc9bed962dd36dacd1 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -14,6 +14,9 @@ */ #include +#include +#include + #include "os.h" #include "taos.h" #include "taosdef.h" @@ -329,6 +332,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'N': arguments->data_batch = atoi(arg); + if (arguments->data_batch >= INT16_MAX) { + arguments->data_batch = INT16_MAX - 1; + } break; case 'L': { @@ -366,6 +372,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { static struct argp argp = {options, parse_opt, args_doc, doc}; static resultStatistics g_resultStatistics = {0}; static FILE *g_fpOfResult = NULL; +static int g_numOfCores = 1; int taosDumpOut(struct arguments *arguments); int taosDumpIn(struct arguments *arguments); @@ -378,7 +385,7 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); int taosCheckParam(struct arguments *arguments); void taosFreeDbInfos(); -static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName); +static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); struct arguments tsArguments = { // connection option @@ -540,6 +547,8 @@ int main(int argc, char *argv[]) { } } + g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); + time_t tTime = time(NULL); struct tm tm = *localtime(&tTime); @@ -692,64 +701,97 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu sprintf(tmpCommand, "select tbname from %s", metric); - TAOS_RES *result = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(result); + TAOS_RES *res = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(res); if (code != 0) { fprintf(stderr, "failed to run command %s\n", tmpCommand); free(tmpCommand); - taos_free_result(result); + taos_free_result(res); return -1; } + free(tmpCommand); - TAOS_FIELD *fields = taos_fetch_fields(result); + char tmpBuf[TSDB_FILENAME_LEN + 1]; + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".select-tbname.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + taos_free_result(res); + return -1; + } - int32_t numOfTable = 0; - int32_t numOfThread = *totalNumOfThread; - char tmpFileName[TSDB_FILENAME_LEN + 1]; - while ((row = taos_fetch_row(result)) != NULL) { - if (0 == numOfTable) { - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, ".tables.tmp.%d", numOfThread); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(result); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - (void)remove(tmpFileName); - } - free(tmpCommand); - return -1; - } - - numOfThread++; - } + TAOS_FIELD *fields = taos_fetch_fields(res); + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { + memset(&tableRecord, 0, sizeof(STableRecord)); tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - + + taosWrite(fd, &tableRecord, sizeof(STableRecord)); numOfTable++; + } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); + + int maxThreads = arguments->thread_num; + int tableOfPerFile ; + if (numOfTable <= arguments->thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / arguments->thread_num; + if (0 != numOfTable % arguments->thread_num) { + tableOfPerFile += 1; + } + } - if (numOfTable >= arguments->table_batch) { - numOfTable = 0; + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; + } + + int32_t numOfThread = *totalNumOfThread; + int subFd = -1; + for (; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); close(fd); - fd = -1; + return -1; + } + + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; } + taosWrite(subFd, tblBuf, readLen); + close(subFd); } + + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); if (fd >= 0) { close(fd); fd = -1; - } - - taos_free_result(result); + } *totalNumOfThread = numOfThread; - - free(tmpCommand); return 0; } @@ -946,7 +988,7 @@ int taosDumpOut(struct arguments *arguments) { } // start multi threads to dumpout - taosStartDumpOutWorkThreads(arguments, totalNumOfThread, dbInfos[0]->name); + taosStartDumpOutWorkThreads(taos, arguments, totalNumOfThread, dbInfos[0]->name); char tmpFileName[TSDB_FILENAME_LEN + 1]; _clean_tmp_file: @@ -1181,34 +1223,34 @@ void* taosDumpOutWorkThreadFp(void *arg) STableRecord tableRecord; int fd; - char tmpFileName[TSDB_FILENAME_LEN*4] = {0}; - sprintf(tmpFileName, ".tables.tmp.%d", pThread->threadIndex); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + char tmpBuf[TSDB_FILENAME_LEN*4] = {0}; + sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); + fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd == -1) { - fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpFileName); + fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpBuf); return NULL; } FILE *fp = NULL; - memset(tmpFileName, 0, TSDB_FILENAME_LEN + 128); + memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); if (tsArguments.outpath[0] != 0) { - sprintf(tmpFileName, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); + sprintf(tmpBuf, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); } else { - sprintf(tmpFileName, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); + sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); } - fp = fopen(tmpFileName, "w"); + fp = fopen(tmpBuf, "w"); if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpFileName); + fprintf(stderr, "failed to open file %s\n", tmpBuf); close(fd); return NULL; } - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, "use %s", pThread->dbName); + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, "use %s", pThread->dbName); - TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpFileName); + TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf); int32_t code = taos_errno(tmpResult); if (code != 0) { fprintf(stderr, "invalid database %s\n", pThread->dbName); @@ -1218,6 +1260,9 @@ void* taosDumpOutWorkThreadFp(void *arg) return NULL; } + int fileNameIndex = 1; + int tablesInOneFile = 0; + int64_t lastRowsPrint = 5000000; fprintf(fp, "USE %s;\n\n", pThread->dbName); while (1) { ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); @@ -1228,6 +1273,33 @@ void* taosDumpOutWorkThreadFp(void *arg) // TODO: sum table count and table rows by self pThread->tablesOfDumpOut++; pThread->rowsOfDumpOut += ret; + + if (pThread->rowsOfDumpOut >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from database %s\n", pThread->rowsOfDumpOut, pThread->dbName); + lastRowsPrint += 5000000; + } + + tablesInOneFile++; + if (tablesInOneFile >= tsArguments.table_batch) { + fclose(fp); + tablesInOneFile = 0; + + memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); + if (tsArguments.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); + } else { + sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex); + } + fileNameIndex++; + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + fprintf(stderr, "failed to open file %s\n", tmpBuf); + close(fd); + taos_free_result(tmpResult); + return NULL; + } + } } } @@ -1238,7 +1310,7 @@ void* taosDumpOutWorkThreadFp(void *arg) return NULL; } -static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName) +static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName) { pthread_attr_t thattr; SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); @@ -1249,12 +1321,7 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfTh pThread->threadIndex = t; pThread->totalThreads = numOfThread; tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN); - pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); - - if (pThread->taosCon == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL)); - exit(0); - } + pThread->taosCon = taosCon; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -1273,7 +1340,6 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfTh int64_t totalRowsOfDumpOut = 0; int64_t totalChildTblsOfDumpOut = 0; for (int32_t t = 0; t < numOfThread; ++t) { - taos_close(threadObj[t].taosCon); totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; } @@ -1398,44 +1464,81 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao return -1; } - TAOS_FIELD *fields = taos_fetch_fields(res); - - int32_t numOfTable = 0; - int32_t numOfThread = 0; - char tmpFileName[TSDB_FILENAME_LEN + 1]; - while ((row = taos_fetch_row(res)) != NULL) { - if (0 == numOfTable) { - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, ".tables.tmp.%d", numOfThread); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(res); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - (void)remove(tmpFileName); - } - return -1; - } + char tmpBuf[TSDB_FILENAME_LEN + 1]; + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".show-tables.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + taos_free_result(res); + return -1; + } - numOfThread++; - } + TAOS_FIELD *fields = taos_fetch_fields(res); + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); - + taosWrite(fd, &tableRecord, sizeof(STableRecord)); - + numOfTable++; + } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); - if (numOfTable >= arguments->table_batch) { - numOfTable = 0; + int maxThreads = tsArguments.thread_num; + int tableOfPerFile ; + if (numOfTable <= tsArguments.thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / tsArguments.thread_num; + if (0 != numOfTable % tsArguments.thread_num) { + tableOfPerFile += 1; + } + } + + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; + } + + int32_t numOfThread = 0; + int subFd = -1; + for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, TSDB_FILENAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".show-tables.tmp"); + (void)remove(tmpBuf); close(fd); - fd = -1; + return -1; } + + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; + } + taosWrite(subFd, tblBuf, readLen); + close(subFd); } + sprintf(tmpBuf, ".show-tables.tmp"); + (void)remove(tmpBuf); + if (fd >= 0) { close(fd); fd = -1; @@ -1444,10 +1547,10 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao taos_free_result(res); // start multi threads to dumpout - taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name); + taosStartDumpOutWorkThreads(taosCon, arguments, numOfThread, dbInfo->name); for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - (void)remove(tmpFileName); + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); } return 0; @@ -1552,8 +1655,8 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols } int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName) { - /* char temp[MAX_COMMAND_SIZE] = "\0"; */ - int64_t totalRows = 0; + int64_t lastRowsPrint = 5000000; + int64_t totalRows = 0; int count = 0; char *pstr = NULL; TAOS_ROW row = NULL; @@ -1680,9 +1783,14 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ") "); - totalRows++; + totalRows++; count++; fprintf(fp, "%s", tmpBuffer); + + if (totalRows >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from %s.%s\n", totalRows, dbName, tbname); + lastRowsPrint += 5000000; + } total_sqlstr_len += curr_sqlstr_len; @@ -2048,6 +2156,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c return -1; } + int lastRowsPrint = 5000000; int lineNo = 0; while ((read_len = getline(&line, &line_len, fp)) != -1) { ++lineNo; @@ -2074,7 +2183,12 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c } memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); - cmd_len = 0; + cmd_len = 0; + + if (lineNo >= lastRowsPrint) { + printf(" %d lines already be executed from file %s\n", lineNo, fileName); + lastRowsPrint += 5000000; + } } tfree(cmd); @@ -2101,7 +2215,7 @@ void* taosDumpInWorkThreadFp(void *arg) return NULL; } -static void taosStartDumpInWorkThreads(struct arguments *args) +static void taosStartDumpInWorkThreads(void* taosCon, struct arguments *args) { pthread_attr_t thattr; SThreadParaObj *pThread; @@ -2116,11 +2230,7 @@ static void taosStartDumpInWorkThreads(struct arguments *args) pThread = threadObj + t; pThread->threadIndex = t; pThread->totalThreads = totalThreads; - pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); - if (pThread->taosCon == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL)); - exit(0); - } + pThread->taosCon = taosCon; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -2169,7 +2279,7 @@ int taosDumpIn(struct arguments *arguments) { taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile); } - taosStartDumpInWorkThreads(arguments); + taosStartDumpInWorkThreads(taos, arguments); taos_close(taos); taosFreeSQLFiles(); diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 6d3061c4269f54f0075a07ee75115f43e856479b..a07607e615a46c34c1a570cacc42c8f36769b754 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -128,8 +128,8 @@ typedef struct { typedef struct { int32_t dnodeId; int8_t role; - int8_t reserved[3]; - SDnodeObj* pDnode; + int8_t vver[3]; // To ensure compatibility, 3 bits are used to represent the remainder of 64 bit version + SDnodeObj *pDnode; } SVnodeGid; typedef struct SVgObj { @@ -144,7 +144,8 @@ typedef struct SVgObj { int8_t status; int8_t reserved0[4]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; - int8_t reserved1[12]; + int32_t vgCfgVersion; + int8_t reserved1[8]; int8_t updateEnd[4]; int32_t refCount; int32_t numOfTables; @@ -173,7 +174,8 @@ typedef struct { int8_t replications; int8_t quorum; int8_t update; - int8_t reserved[11]; + int8_t cacheLastRow; + int8_t reserved[10]; } SDbCfg; typedef struct SDbObj { @@ -181,7 +183,7 @@ typedef struct SDbObj { int8_t reserved0[4]; char acct[TSDB_USER_LEN]; int64_t createdTime; - int32_t cfgVersion; + int32_t dbCfgVersion; SDbCfg cfg; int8_t status; int8_t reserved1[11]; diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index 8bc29ef9efabf73470896b49b86aad276a29a748..56d7455ad24f76a0b289e8ad8882db1867d0bed9 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -52,15 +52,18 @@ typedef enum EDnodeOfflineReason { TAOS_DN_OFF_TIME_ZONE_NOT_MATCH, TAOS_DN_OFF_LOCALE_NOT_MATCH, TAOS_DN_OFF_CHARSET_NOT_MATCH, + TAOS_DN_OFF_FLOW_CTRL_NOT_MATCH, + TAOS_DN_OFF_SLAVE_QUERY_NOT_MATCH, + TAOS_DN_OFF_ADJUST_MASTER_NOT_MATCH, TAOS_DN_OFF_OTHERS } EDnodeOfflineReason; +extern char* dnodeStatus[]; +extern char* dnodeRoles[]; + int32_t mnodeInitDnodes(); void mnodeCleanupDnodes(); -char* mnodeGetDnodeStatusStr(int32_t dnodeStatus); -void mgmtMonitorDnodeModule(); - int32_t mnodeGetDnodesNum(); int32_t mnodeGetOnlinDnodesCpuCoreNum(); int32_t mnodeGetOnlineDnodesNum(); diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index 93f2fa11ea95f22ca4addf12496b79cdaddbf290..ffdec02eb6ca3fcb2b6cd230fb93ebb26dea26de 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -43,8 +43,8 @@ void mnodeIncMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode); char * mnodeGetMnodeRoleStr(); -void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet); -void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect); +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect); char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 31ea2da640ef3b20b84ca769a22637332d6ffe34..e4df562d81c7ef91fe1ff62eba6611d76e0a3ff1 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -59,7 +59,7 @@ typedef struct SSdbRow { SMnodeMsg *pMsg; int32_t (*fpReq)(SMnodeMsg *pMsg); int32_t (*fpRsp)(SMnodeMsg *pMsg, int32_t code); - char reserveForSync[16]; + char reserveForSync[24]; SWalHead pHead[]; } SSdbRow; diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index ee9ec7ae9305c3d436b6e5bec0cd4e94ecc1adf4..2067ad04ccf963701bffcf134c8ae478547f1abb 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -53,6 +53,9 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromIp(char *ep); +int32_t mnodeGetVgidVer(int8_t *vver); +void mnodeSetVgidVer(int8_t *cver, uint64_t iver); + #ifdef __cplusplus } #endif diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 56229daffa87d3300cc597d9353b5a720751a0f9..a35e30481077d0e16888d0ee036312cd068ea9b4 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -171,7 +171,7 @@ void mnodeUpdateClusterId() { void *pIter = mnodeGetNextCluster(NULL, &pCluster); if (pCluster != NULL) { tstrncpy(tsClusterId, pCluster->uid, TSDB_CLUSTER_ID_LEN); - mInfo("cluster id is set to %s", tsClusterId); + mDebug("cluster id is set to %s", tsClusterId); } mnodeDecClusterRef(pCluster); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 25dbb10536c5d0d784b5cbd5624dfca322321c5d..333844ccbea801f6ad0bd3c31bdf0ee7f9287c85 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -322,6 +322,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { return TSDB_CODE_MND_INVALID_DB_OPTION; } + if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { + mError("invalid db option cacheLastRow:%d valid range: [%d, %d]", pCfg->cacheLastRow, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + return TSDB_CODE_SUCCESS; } @@ -343,6 +348,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->replications < 0) pCfg->replications = tsReplications; if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; if (pCfg->update < 0) pCfg->update = tsUpdate; + if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; } static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { @@ -396,7 +402,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * .walLevel = pCreate->walLevel, .replications = pCreate->replications, .quorum = pCreate->quorum, - .update = pCreate->update + .update = pCreate->update, + .cacheLastRow = pCreate->cacheLastRow }; mnodeSetDefaultDbCfg(&pDb->cfg); @@ -605,6 +612,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn strcpy(pSchema[cols].name, "comp"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "cachelast"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; #ifndef __CLOUD_VERSION__ } #endif @@ -750,6 +763,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int8_t *)pWrite = pDb->cfg.compression; cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.cacheLastRow; + cols++; #ifndef __CLOUD_VERSION__ } #endif @@ -864,6 +881,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { int8_t quorum = pAlter->quorum; int8_t precision = pAlter->precision; int8_t update = pAlter->update; + int8_t cacheLastRow = pAlter->cacheLastRow; terrno = TSDB_CODE_SUCCESS; @@ -976,6 +994,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { #endif } + if (cacheLastRow >= 0 && cacheLastRow != pDb->cfg.cacheLastRow) { + mDebug("db:%s, cacheLastRow:%d change to %d", pDb->name, pDb->cfg.cacheLastRow, cacheLastRow); + newCfg.cacheLastRow = cacheLastRow; + } + return newCfg; } @@ -1015,7 +1038,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) { pDb->cfg = newCfg; - pDb->cfgVersion++; + pDb->dbCfgVersion++; SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsDbSdb, diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 037ee2864a2594320028a68afe4a0e1de28b7a69..14d1fa5816ffc168463e0392e675e20cc783af7c 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -63,7 +63,6 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); -static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole); static void mnodeUpdateDnodeEps(); static char* offlineReason[] = { @@ -104,7 +103,7 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) { dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort); mnodeUpdateDnodeEps(); - mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); + mInfo("dnode:%d, fqdn:%s ep:%s port:%d is created", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); return TSDB_CODE_SUCCESS; } @@ -376,10 +375,6 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { mError("\"numOfMnodes\"[%d - %d] cfg parameters inconsistent", clusterCfg->numOfMnodes, htonl(tsNumOfMnodes)); return TAOS_DN_OFF_NUM_OF_MNODES_NOT_MATCH; } - if (clusterCfg->enableBalance != htonl(tsEnableBalance)) { - mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, htonl(tsEnableBalance)); - return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH; - } if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) { mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg parameters inconsistent", clusterCfg->mnodeEqualVnodeNum, htonl(tsMnodeEqualVnodeNum)); @@ -429,6 +424,23 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { return TAOS_DN_OFF_CHARSET_NOT_MATCH; } + if (clusterCfg->enableBalance != tsEnableBalance) { + mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, tsEnableBalance); + return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH; + } + if (clusterCfg->flowCtrl != tsEnableFlowCtrl) { + mError("\"flowCtrl\"[%d - %d] cfg parameters inconsistent", clusterCfg->flowCtrl, tsEnableFlowCtrl); + return TAOS_DN_OFF_FLOW_CTRL_NOT_MATCH; + } + if (clusterCfg->slaveQuery != tsEnableSlaveQuery) { + mError("\"slaveQuery\"[%d - %d] cfg parameters inconsistent", clusterCfg->slaveQuery, tsEnableSlaveQuery); + return TAOS_DN_OFF_SLAVE_QUERY_NOT_MATCH; + } + if (clusterCfg->adjustMaster != tsEnableAdjustMaster) { + mError("\"adjustMaster\"[%d - %d] cfg parameters inconsistent", clusterCfg->adjustMaster, tsEnableAdjustMaster); + return TAOS_DN_OFF_ADJUST_MASTER_NOT_MATCH; + } + return 0; } @@ -557,7 +569,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { for (int32_t j = 0; j < openVnodes; ++j) { SVnodeLoad *pVload = &pStatus->load[j]; pVload->vgId = htonl(pVload->vgId); - pVload->cfgVersion = htonl(pVload->cfgVersion); + pVload->dbCfgVersion = htonl(pVload->dbCfgVersion); + pVload->vgCfgVersion = htonl(pVload->vgCfgVersion); + pVload->vnodeVersion = htobe64(pVload->vnodeVersion); SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId); if (pVgroup == NULL) { @@ -833,12 +847,12 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char* status = mnodeGetDnodeStatusStr(pDnode->status); + char* status = dnodeStatus[pDnode->status]; STR_TO_VARSTR(pWrite, status); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char* role = mnodeGetDnodeAlternativeRoleStr(pDnode->alternativeRole); + char* role = dnodeRoles[pDnode->alternativeRole]; STR_TO_VARSTR(pWrite, role); cols++; @@ -1031,6 +1045,11 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int8_t *)cfg->ptr)); + varDataSetLen(pWrite, t); + numOfRows++; + break; case TAOS_CFG_VTYPE_INT16: t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); varDataSetLen(pWrite, t); @@ -1154,21 +1173,17 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo return numOfRows; } -char* mnodeGetDnodeStatusStr(int32_t dnodeStatus) { - switch (dnodeStatus) { - case TAOS_DN_STATUS_OFFLINE: return "offline"; - case TAOS_DN_STATUS_DROPPING: return "dropping"; - case TAOS_DN_STATUS_BALANCING: return "balancing"; - case TAOS_DN_STATUS_READY: return "ready"; - default: return "undefined"; - } -} +char* dnodeStatus[] = { + "offline", + "dropping", + "balancing", + "ready", + "undefined" +}; -static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) { - switch (alternativeRole) { - case TAOS_DN_ALTERNATIVE_ROLE_ANY: return "any"; - case TAOS_DN_ALTERNATIVE_ROLE_MNODE: return "mnode"; - case TAOS_DN_ALTERNATIVE_ROLE_VNODE: return "vnode"; - default:return "any"; - } -} +char* dnodeRoles[] = { + "any", + "mnode", + "vnode", + "any" +}; diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 7b520c6022bd3b38450657d4a0fa68fc3cfcc581..6e001f4dfbb64c9d73581fef624b4b763b39d50c 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -91,6 +91,7 @@ int32_t mnodeStartSystem() { return -1; } + dnodeReportStep("mnode-grant", "start to set grant infomation", 0); grantReset(TSDB_GRANT_ALL, 0); tsMgmtIsRunning = true; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 68acae7dec0e5505e72733a638abcc142eeeb407..3ea41c41c6ffe89671853387e59733f7db7506cd 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -72,7 +72,7 @@ static int32_t mnodeMnodeActionInsert(SSdbRow *pRow) { pDnode->isMgmt = true; mnodeDecDnodeRef(pDnode); - mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, + mInfo("mnode:%d, fqdn:%s ep:%s port:%u is created", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort); return TSDB_CODE_SUCCESS; } @@ -202,13 +202,13 @@ void mnodeCancelGetNextMnode(void *pIter) { void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { bool set = false; SMInfos mInfos = {0}; - mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); if (pMinfos != NULL) { + mInfo("vgId:1, update mnodes epSet, numOfMinfos:%d", pMinfos->mnodeNum); set = true; mInfos = *pMinfos; - } - else { + } else { + mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d", mnodeGetMnodesNum()); int32_t index = 0; void * pIter = NULL; while (1) { @@ -273,14 +273,14 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { mnodeMnodeUnLock(); } -void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet, bool redirect) { mnodeMnodeRdLock(); *epSet = tsMEpForPeer; mnodeMnodeUnLock(); mTrace("vgId:1, mnodes epSet for peer is returned, num:%d inUse:%d", tsMEpForPeer.numOfEps, tsMEpForPeer.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { - if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { + if (redirect && strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { epSet->inUse = (i + 1) % epSet->numOfEps; mTrace("vgId:1, mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { @@ -289,14 +289,19 @@ void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { } } -void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet, bool redirect) { mnodeMnodeRdLock(); *epSet = tsMEpForShell; mnodeMnodeUnLock(); + if (mnodeGetDnodesNum() <= 1) { + epSet->numOfEps = 0; + return; + } + mTrace("vgId:1, mnodes epSet for shell is returned, num:%d inUse:%d", tsMEpForShell.numOfEps, tsMEpForShell.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { - if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { + if (redirect && strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { epSet->inUse = (i + 1) % epSet->numOfEps; mTrace("vgId:1, mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { @@ -372,6 +377,25 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) { return code; } +static bool mnodeAllOnline() { + void *pIter = NULL; + bool allOnline = true; + + while (1) { + SMnodeObj *pMnode = NULL; + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + if (pMnode->role != TAOS_SYNC_ROLE_MASTER && pMnode->role != TAOS_SYNC_ROLE_SLAVE) { + allOnline = false; + mDebug("mnode:%d, role:%s, not online", pMnode->mnodeId, syncRole[pMnode->role]); + mnodeDecMnodeRef(pMnode); + } + } + mnodeCancelGetNextMnode(pIter); + + return allOnline; +} + void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj)); pMnode->mnodeId = dnodeId; @@ -384,6 +408,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { .fpRsp = mnodeCreateMnodeCb }; + if (needConfirm && !mnodeAllOnline()) { + mDebug("wait all mnode online then create new mnode"); + return; + } + int32_t code = TSDB_CODE_SUCCESS; if (needConfirm) { code = mnodeSendCreateMnodeMsg(dnodeId, dnodeEp); diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index cfb7b7781b04c0145f66a58c6d3d2eaefa0913f3..aaf8b694279299215dbfe386755c43ed2200e555 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -54,7 +54,7 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); - mnodeGetMnodeEpSetForPeer(epSet); + mnodeGetMnodeEpSetForPeer(epSet, true); rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 5d63ae9ff411cdddd82400d1f2853e6e882ba527..7c35829f88f4e7d1753a3ee00b13a9f87d03ae48 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -282,10 +282,11 @@ static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, voi // not thread safe, need optimized int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) { - pConn->numOfQueries = 0; + pConn->numOfQueries = 0; pConn->numOfStreams = 0; - int32_t numOfQueries = htonl(pHBMsg->numOfQueries); + int32_t numOfStreams = htonl(pHBMsg->numOfStreams); + if (numOfQueries > 0) { if (pConn->pQueries == NULL) { pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE); @@ -299,7 +300,6 @@ int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) { } } - int32_t numOfStreams = htonl(pHBMsg->numOfStreams); if (numOfStreams > 0) { if (pConn->pStreams == NULL) { pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE); @@ -309,7 +309,7 @@ int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pHBMsg) { int32_t saveSize = pConn->numOfStreams * sizeof(SStreamDesc); if (saveSize > 0 && pConn->pStreams != NULL) { - memcpy(pConn->pStreams, pHBMsg->pData + pConn->numOfQueries * sizeof(SQueryDesc), saveSize); + memcpy(pConn->pStreams, pHBMsg->pData + numOfQueries * sizeof(SQueryDesc), saveSize); } } diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index c2a70bc01d2194fff5b8095d06e40535b940dc25..200f589b7866d3fa6cf9865be397935657d7ba86 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -50,7 +50,7 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); - mnodeGetMnodeEpSetForShell(epSet); + mnodeGetMnodeEpSetForShell(epSet, true); rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6cc4e097350783a93300c46adbd5f68faf43e143..6997d0a666555e1237b7f5e019368c822ed2a6b1 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -183,18 +183,23 @@ static int32_t sdbInitWal() { return -1; } - sdbInfo("vgId:1, open wal for restore"); + sdbInfo("vgId:1, open sdb wal for restore"); int32_t code = walRestore(tsSdbMgmt.wal, NULL, sdbProcessWrite); if (code != TSDB_CODE_SUCCESS) { sdbError("vgId:1, failed to open wal for restore since %s", tstrerror(code)); return -1; } + + sdbInfo("vgId:1, sdb wal load success"); return 0; } static void sdbRestoreTables() { int32_t totalRows = 0; int32_t numOfTables = 0; + + sdbInfo("vgId:1, sdb start to check for integrity"); + for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) { SSdbTable *pTable = sdbGetTableFromId(tableId); if (pTable == NULL) continue; @@ -204,7 +209,7 @@ static void sdbRestoreTables() { totalRows += pTable->numOfRows; numOfTables++; - sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->name, pTable->numOfRows); + sdbInfo("vgId:1, sdb:%s is checked, rows:%" PRId64, pTable->name, pTable->numOfRows); } sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbMgmt.version, totalRows, numOfTables); @@ -220,6 +225,10 @@ void sdbUpdateMnodeRoles() { for (int32_t i = 0; i < tsSdbMgmt.cfg.replica; ++i) { SMnodeObj *pMnode = mnodeGetMnode(roles.nodeId[i]); if (pMnode != NULL) { + if (pMnode->role != roles.role[i]) { + bnNotify(); + } + pMnode->role = roles.role[i]; sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]); if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role; @@ -628,6 +637,12 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * SSdbTable *pTable = sdbGetTableFromId(tableId); assert(pTable != NULL); + if (!mnodeIsRunning() && tsSdbMgmt.version % 100000 == 0) { + char stepDesc[TSDB_STEP_DESC_LEN] = {0}; + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "%" PRIu64 " rows have been restored", tsSdbMgmt.version); + dnodeReportStep("mnode-sdb", stepDesc, 0); + } + if (qtype == TAOS_QTYPE_QUERY) return sdbPerformDeleteAction(pHead, pTable); pthread_mutex_lock(&tsSdbMgmt.mutex); @@ -1070,6 +1085,8 @@ static void *sdbWorkerFp(void *pWorker) { int32_t qtype; void * unUsed; + taosBlockSIGPIPE(); + while (1) { int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); if (numOfMsgs == 0) { diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 2da46d5b4bd01d47794b0e77011101ce7dc137cc..6b9f0e26a762092cfae02b5724d6cfedd24194d2 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -282,7 +282,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); pRsp->totalDnodes = htonl(mnodeGetDnodesNum()); - mnodeGetMnodeEpSetForShell(&pRsp->epSet); + mnodeGetMnodeEpSetForShell(&pRsp->epSet, false); pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.len = sizeof(SHeartBeatRsp); @@ -349,7 +349,9 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet); + mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet, false); + + dnodeGetClusterId(pConnectRsp->clusterId); connect_over: if (code != TSDB_CODE_SUCCESS) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 6297bb21d0ba45ba0363c6d7de2865e62c535014..26e60cdc9cfae17a53e14ad3041ffc058883d1fb 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -26,6 +26,7 @@ #include "tcompare.h" #include "tdataformat.h" #include "tgrant.h" +#include "tqueue.h" #include "hash.h" #include "mnode.h" #include "dnode.h" @@ -49,12 +50,14 @@ #define CREATE_CTABLE_RETRY_TIMES 10 #define CREATE_CTABLE_RETRY_SEC 14 -int64_t tsCTableRid = -1; -static void * tsChildTableSdb; -int64_t tsSTableRid = -1; -static void * tsSuperTableSdb; -static int32_t tsChildTableUpdateSize; -static int32_t tsSuperTableUpdateSize; +int64_t tsCTableRid = -1; +static void * tsChildTableSdb; +int64_t tsSTableRid = -1; +static void * tsSuperTableSdb; +static SHashObj *tsSTableUidHash; +static int32_t tsChildTableUpdateSize; +static int32_t tsSuperTableUpdateSize; + static void * mnodeGetChildTable(char *tableId); static void * mnodeGetSuperTable(char *tableId); static void * mnodeGetSuperTableByUid(uint64_t uid); @@ -289,6 +292,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { } static int32_t mnodeChildTableActionRestored() { +#if 0 void *pIter = NULL; SCTableObj *pTable = NULL; @@ -345,6 +349,7 @@ static int32_t mnodeChildTableActionRestored() { } mnodeCancelGetNextChildTable(pIter); +#endif return 0; } @@ -447,6 +452,7 @@ static int32_t mnodeSuperTableActionInsert(SSdbRow *pRow) { } mnodeDecDbRef(pDb); + taosHashPut(tsSTableUidHash, &pStable->uid, sizeof(int64_t), &pStable, sizeof(int64_t)); return TSDB_CODE_SUCCESS; } @@ -459,6 +465,7 @@ static int32_t mnodeSuperTableActionDelete(SSdbRow *pRow) { } mnodeDecDbRef(pDb); + taosHashRemove(tsSTableUidHash, &pStable->uid, sizeof(int64_t)); return TSDB_CODE_SUCCESS; } @@ -570,6 +577,7 @@ static int32_t mnodeInitSuperTables() { .fpRestored = mnodeSuperTableActionRestored }; + tsSTableUidHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); tsSTableRid = sdbOpenTable(&desc); tsSuperTableSdb = sdbGetTableByRid(tsSTableRid); if (tsSuperTableSdb == NULL) { @@ -584,6 +592,9 @@ static int32_t mnodeInitSuperTables() { static void mnodeCleanupSuperTables() { sdbCloseTable(tsSTableRid); tsSuperTableSdb = NULL; + + taosHashCleanup(tsSTableUidHash); + tsSTableUidHash = NULL; } int32_t mnodeInitTables() { @@ -633,20 +644,12 @@ static void *mnodeGetSuperTable(char *tableId) { } static void *mnodeGetSuperTableByUid(uint64_t uid) { - SSTableObj *pStable = NULL; - void *pIter = NULL; - - while (1) { - pIter = mnodeGetNextSuperTable(pIter, &pStable); - if (pStable == NULL) break; - if (pStable->uid == uid) { - mnodeCancelGetNextSuperTable(pIter); - return pStable; - } - mnodeDecTableRef(pStable); - } + SSTableObj **ppStable = taosHashGet(tsSTableUidHash, &uid, sizeof(int64_t)); + if (ppStable == NULL || *ppStable == NULL) return NULL; - return NULL; + SSTableObj *pStable = *ppStable; + mnodeIncTableRef(pStable); + return pStable; } void *mnodeGetTable(char *tableId) { @@ -718,12 +721,151 @@ static void mnodeExtractTableName(char* tableId, char* name) { } } +static SMnodeMsg *mnodeCreateSubMsg(SMnodeMsg *pBatchMasterMsg, int32_t contSize) { + SMnodeMsg *pSubMsg = taosAllocateQitem(sizeof(*pBatchMasterMsg) + contSize); + *pSubMsg = *pBatchMasterMsg; + + //pSubMsg->pCont = (char *) pSubMsg + sizeof(SMnodeMsg); + pSubMsg->rpcMsg.pCont = pSubMsg->pCont; + pSubMsg->successed = 0; + pSubMsg->expected = 0; + SCMCreateTableMsg *pCM = pSubMsg->rpcMsg.pCont; + pCM->numOfTables = htonl(1); + pCM->contLen = htonl(contSize); + + return pSubMsg; +} + +void mnodeDestroySubMsg(SMnodeMsg *pSubMsg) { + if (pSubMsg) { + // pUser is retained in batch master msg + if (pSubMsg->pDb) mnodeDecDbRef(pSubMsg->pDb); + if (pSubMsg->pVgroup) mnodeDecVgroupRef(pSubMsg->pVgroup); + if (pSubMsg->pTable) mnodeDecTableRef(pSubMsg->pTable); + if (pSubMsg->pSTable) mnodeDecTableRef(pSubMsg->pSTable); + if (pSubMsg->pAcct) mnodeDecAcctRef(pSubMsg->pAcct); + if (pSubMsg->pDnode) mnodeDecDnodeRef(pSubMsg->pDnode); + + taosFreeQitem(pSubMsg); + } +} + +static int32_t mnodeValidateCreateTableMsg(SCreateTableMsg *pCreateTable, SMnodeMsg *pMsg) { + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreateTable->db); + if (pMsg->pDb == NULL) { + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + return TSDB_CODE_MND_DB_NOT_SELECTED; + } + + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { + mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); + return TSDB_CODE_MND_DB_IN_DROPPING; + } + + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreateTable->tableId); + if (pMsg->pTable != NULL && pMsg->retry == 0) { + if (pCreateTable->getMeta) { + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + return mnodeGetChildTableMeta(pMsg); + } else if (pCreateTable->igExists) { + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + return TSDB_CODE_SUCCESS; + } else { + mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableId); + return TSDB_CODE_MND_TABLE_ALREADY_EXIST; + } + } + + if (pCreateTable->numOfTags != 0) { + mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableId, pMsg->rpcMsg.handle); + return mnodeProcessCreateSuperTableMsg(pMsg); + } else { + mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableId, pMsg->rpcMsg.handle); + return mnodeProcessCreateChildTableMsg(pMsg); + } +} + +static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) { + if (pMsg->pBatchMasterMsg == NULL) { // batch master first round + pMsg->pBatchMasterMsg = pMsg; + + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + int32_t numOfTables = htonl(pCreate->numOfTables); + int32_t contentLen = htonl(pCreate->contLen); + pMsg->expected = numOfTables; + + int32_t code = TSDB_CODE_SUCCESS; + SCreateTableMsg *pCreateTable = (SCreateTableMsg*) ((char*) pCreate + sizeof(SCMCreateTableMsg)); + for (SCreateTableMsg *p = pCreateTable; p < (SCreateTableMsg *) ((char *) pCreate + contentLen); p = (SCreateTableMsg *) ((char *) p + htonl(p->len))) { + SMnodeMsg *pSubMsg = mnodeCreateSubMsg(pMsg, sizeof(SCMCreateTableMsg) + htonl(p->len)); + memcpy(pSubMsg->pCont + sizeof(SCMCreateTableMsg), p, htonl(p->len)); + code = mnodeValidateCreateTableMsg(p, pSubMsg); + + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + ++pSubMsg->pBatchMasterMsg->successed; + mnodeDestroySubMsg(pSubMsg); + continue; + } + + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mnodeDestroySubMsg(pSubMsg); + return code; + } + } + + if (pMsg->successed >= pMsg->expected) { + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + } else { + if (pMsg->pBatchMasterMsg != pMsg) { // batch sub replay + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateTableMsg *pCreateTable = (SCreateTableMsg*) ((char*) pCreate + sizeof(SCMCreateTableMsg)); + int32_t code = mnodeValidateCreateTableMsg(pCreateTable, pMsg); + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + ++pMsg->pBatchMasterMsg->successed; + mnodeDestroySubMsg(pMsg); + } + + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mnodeDestroySubMsg(pMsg); + return code; + } + + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + } else { // batch master replay, reprocess the whole batch + assert(0); + } + } +} + static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; - - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db); + + int32_t numOfTables = htonl(pCreate->numOfTables); + int32_t contentLen = htonl(pCreate->contLen); + if (numOfTables == 0 || contentLen == 0) { + // todo return error + } + + // batch master msg first round or reprocessing and batch sub msg reprocessing + if (numOfTables > 1 || pMsg->pBatchMasterMsg != NULL) { + return mnodeProcessBatchCreateTableMsg(pMsg); + } + + SCreateTableMsg *p = (SCreateTableMsg*)((char*) pCreate + sizeof(SCMCreateTableMsg)); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(p->db); if (pMsg->pDb == NULL) { - mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, p->tableId); return TSDB_CODE_MND_DB_NOT_SELECTED; } @@ -732,28 +874,28 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_DB_IN_DROPPING; } - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(p->tableId); if (pMsg->pTable != NULL && pMsg->retry == 0) { - if (pCreate->getMeta) { - mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + if (p->getMeta) { + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, p->tableId); return mnodeGetChildTableMeta(pMsg); - } else if (pCreate->igExists) { - mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + } else if (p->igExists) { + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, p->tableId); return TSDB_CODE_SUCCESS; } else { mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId); + p->tableId); return TSDB_CODE_MND_TABLE_ALREADY_EXIST; } } - if (pCreate->numOfTags != 0) { + if (p->numOfTags != 0) { mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, pMsg->rpcMsg.handle); + p->tableId, pMsg->rpcMsg.handle); return mnodeProcessCreateSuperTableMsg(pMsg); } else { mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, pMsg->rpcMsg.handle); + p->tableId, pMsg->rpcMsg.handle); return mnodeProcessCreateChildTableMsg(pMsg); } } @@ -859,7 +1001,13 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + SCMCreateTableMsg *pCreate1 = pMsg->rpcMsg.pCont; + if (pCreate1->numOfTables == 0) { + // todo return to error message + } + + SCreateTableMsg* pCreate = (SCreateTableMsg*)((char*)pCreate1 + sizeof(SCMCreateTableMsg)); + SSTableObj * pStable = calloc(1, sizeof(SSTableObj)); if (pStable == NULL) { mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); @@ -1599,8 +1747,11 @@ static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg) { mInfo("drop stable rsp received, result:%s", tstrerror(rpcMsg->code)); } -static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj *pTable) { - STagData * pTagData = NULL; +static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pCreateMsg, SCTableObj *pTable) { + SCreateTableMsg* pMsg = (SCreateTableMsg*) ((char*)pCreateMsg + sizeof(SCMCreateTableMsg)); + + char* tagData = NULL; + int32_t tagDataLen = 0; int32_t totalCols = 0; int32_t contLen = 0; @@ -1608,9 +1759,13 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj * totalCols = pTable->superTable->numOfColumns + pTable->superTable->numOfTags; contLen = sizeof(SMDCreateTableMsg) + totalCols * sizeof(SSchema) + pTable->sqlLen; if (pMsg != NULL) { - pTagData = (STagData *)pMsg->schema; - tagDataLen = htonl(pTagData->dataLen); + int32_t nameLen = htonl(*(int32_t*)pMsg->schema); + char* p = pMsg->schema + nameLen + sizeof(int32_t); + + tagDataLen = htonl(*(int32_t*) p); contLen += tagDataLen; + + tagData = p + sizeof(int32_t); } } else { totalCols = pTable->numOfColumns; @@ -1662,7 +1817,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj * } if (pTable->info.type == TSDB_CHILD_TABLE && pMsg != NULL) { - memcpy(pCreate->data + totalCols * sizeof(SSchema), pTagData->data, tagDataLen); + memcpy(pCreate->data + totalCols * sizeof(SSchema), tagData, tagDataLen); } if (pTable->info.type == TSDB_STREAM_TABLE) { @@ -1700,7 +1855,8 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + + SCreateTableMsg *pCreate = (SCreateTableMsg*) ((char*)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); assert(pTable); if (code == TSDB_CODE_SUCCESS) { @@ -1714,6 +1870,18 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { mDebug("msg:%p, app:%p table:%s, created in dnode, thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->rpcMsg.handle); + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->successed; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -1728,40 +1896,42 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SVgObj *pVgroup = pMsg->pVgroup; - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + + SCMCreateTableMsg *p1 = pMsg->rpcMsg.pCont; + SCreateTableMsg *pCreate = (SCreateTableMsg*)((char*)p1 + sizeof(SCMCreateTableMsg)); + SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) { mError("msg:%p, app:%p table:%s, failed to alloc memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } - if (pCreate->numOfColumns == 0) { - pTable->info.type = TSDB_CHILD_TABLE; - } else { - pTable->info.type = TSDB_NORMAL_TABLE; - } - - pTable->info.tableId = strdup(pCreate->tableId); + pTable->info.type = (pCreate->numOfColumns == 0)? TSDB_CHILD_TABLE:TSDB_NORMAL_TABLE; + pTable->info.tableId = strdup(pCreate->tableId); pTable->createdTime = taosGetTimestampMs(); pTable->tid = tid; pTable->vgId = pVgroup->vgId; if (pTable->info.type == TSDB_CHILD_TABLE) { - STagData *pTagData = (STagData *)pCreate->schema; // it is a tag key + int32_t nameLen = htonl(*(int32_t*) pCreate->schema); + char* name = (char*)pCreate->schema + sizeof(int32_t); + + char stableName[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(stableName, name, nameLen); char prefix[64] = {0}; size_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64); - if (0 != strncasecmp(prefix, pTagData->name, prefixLen)) { + if (0 != strncasecmp(prefix, stableName, prefixLen)) { mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, pTagData->name); + pCreate->tableId, stableName); mnodeDestroyChildTable(pTable); return TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; } - if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name); + if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(stableName); if (pMsg->pSTable == NULL) { mError("msg:%p, app:%p table:%s, corresponding super table:%s does not exist", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, pTagData->name); + pCreate->tableId, stableName); mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -1839,7 +2009,9 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { - SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + //SCMCreateTableMsg* p1 = pMsg->rpcMsg.pCont; // there are several tables here. + SCreateTableMsg* pCreate = (SCreateTableMsg*)(pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); + int32_t code = grantCheck(TSDB_GRANT_TIMESERIES); if (code != TSDB_CODE_SUCCESS) { mError("msg:%p, app:%p table:%s, failed to create, grant timeseries failed", pMsg, pMsg->rpcMsg.ahandle, @@ -2144,11 +2316,12 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); - if (pTable->superTable != NULL) { - tstrncpy(pMeta->sTableId, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); - } - if (pTable->info.type == TSDB_CHILD_TABLE && pTable->superTable != NULL) { + if (pTable->info.type == TSDB_CHILD_TABLE) { + assert(pTable->superTable != NULL); + tstrncpy(pMeta->sTableName, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); + + pMeta->suid = pTable->superTable->uid; pMeta->sversion = htons(pTable->superTable->sversion); pMeta->tversion = htons(pTable->superTable->tversion); pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags; @@ -2187,15 +2360,23 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; - STagData *pTags = (STagData *)pInfo->tags; - int32_t tagLen = htonl(pTags->dataLen); - if (pTags->name[0] == 0) { - mError("msg:%p, app:%p table:%s, failed to create table on demand for stable is empty, tagLen:%d", pMsg, + + char* p = pInfo->tags; + int32_t nameLen = htonl(*(int32_t*) p); + p += sizeof(int32_t); + p += nameLen; + + int32_t tagLen = htonl(*(int32_t*) p); + p += sizeof(int32_t); + + int32_t totalLen = nameLen + tagLen + sizeof(int32_t)*2; + if (tagLen == 0 || nameLen == 0) { + mError("msg:%p, app:%p table:%s, failed to create table on demand for super table is empty, tagLen:%d", pMsg, pMsg->rpcMsg.ahandle, pInfo->tableId, tagLen); return TSDB_CODE_MND_INVALID_STABLE_NAME; } - int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen; + int32_t contLen = sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg) + totalLen; SCMCreateTableMsg *pCreateMsg = calloc(1, contLen); if (pCreateMsg == NULL) { mError("msg:%p, app:%p table:%s, failed to create table while get meta info, no enough memory", pMsg, @@ -2203,16 +2384,24 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - size_t size = sizeof(pInfo->tableId); - tstrncpy(pCreateMsg->tableId, pInfo->tableId, size); - tstrncpy(pCreateMsg->db, pMsg->pDb->name, sizeof(pCreateMsg->db)); - pCreateMsg->igExists = 1; - pCreateMsg->getMeta = 1; + SCreateTableMsg* pCreate = (SCreateTableMsg*) ((char*) pCreateMsg + sizeof(SCMCreateTableMsg)); + + size_t size = tListLen(pInfo->tableId); + tstrncpy(pCreate->tableId, pInfo->tableId, size); + tstrncpy(pCreate->db, pMsg->pDb->name, sizeof(pCreate->db)); + pCreate->igExists = 1; + pCreate->getMeta = 1; + + pCreateMsg->numOfTables = htonl(1); pCreateMsg->contLen = htonl(contLen); - memcpy(pCreateMsg->schema, pTags, contLen - sizeof(SCMCreateTableMsg)); + memcpy(pCreate->schema, pInfo->tags, totalLen); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(name, pInfo->tags + sizeof(int32_t), nameLen); + mDebug("msg:%p, app:%p table:%s, start to create on demand, tagLen:%d stable:%s", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId, tagLen, pTags->name); + pInfo->tableId, tagLen, name); if (pMsg->rpcMsg.pCont != pMsg->pCont) { tfree(pMsg->rpcMsg.pCont); @@ -2434,6 +2623,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeSendDropChildTableMsg(pMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->successed; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); return; } @@ -2451,6 +2653,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { pMsg->pTable = NULL; mnodeDestroyChildTable(pTable); + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, code); } } else { @@ -2476,6 +2691,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { //Avoid retry again in client rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY; } + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); } } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index eec559600fa5ec4b4bd9a4bcb7d2335a2a10b720..827be0687dbac338d660d07fefaf563d5c5d0341 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -184,6 +184,7 @@ static int32_t mnodeVgroupActionEncode(SSdbRow *pRow) { for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { pTmpVgroup->vnodeGid[i].pDnode = NULL; pTmpVgroup->vnodeGid[i].role = 0; + memset(pTmpVgroup->vnodeGid[i].vver, 0, sizeof(pTmpVgroup->vnodeGid[i].vver)); } pRow->rowSize = tsVgUpdateSize; @@ -256,6 +257,8 @@ SVgObj *mnodeGetVgroup(int32_t vgId) { } void mnodeUpdateVgroup(SVgObj *pVgroup) { + pVgroup->vgCfgVersion++; + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsVgroupSdb, @@ -315,9 +318,10 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; if (pVgid->pDnode == pDnode) { - mTrace("dnode:%d, receive status from dnode, vgId:%d status:%s last:%s", pDnode->dnodeId, pVgroup->vgId, - syncRole[pVload->role], syncRole[pVgid->role]); + mTrace("vgId:%d, receive vnode status from dnode:%d, status:%s last:%s vver:%" PRIu64, pVgroup->vgId, + pDnode->dnodeId, syncRole[pVload->role], syncRole[pVgid->role], pVload->vnodeVersion); pVgid->role = pVload->role; + mnodeSetVgidVer(pVgid->vver, pVload->vnodeVersion); if (pVload->role == TAOS_SYNC_ROLE_MASTER) { pVgroup->inUse = i; } @@ -339,10 +343,11 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl pVgroup->pointsWritten = htobe64(pVload->pointsWritten); } - if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) { - mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mnode cfgVersion:%d replica:%d", - pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion, - pVgroup->numOfVnodes); + if (pVload->dbCfgVersion != pVgroup->pDb->dbCfgVersion || pVload->replica != pVgroup->numOfVnodes || + pVload->vgCfgVersion != pVgroup->vgCfgVersion) { + mError("dnode:%d, vgId:%d, vnode cfgVersion:%d:%d repica:%d not match with mnode cfgVersion:%d:%d replica:%d", + pDnode->dnodeId, pVload->vgId, pVload->dbCfgVersion, pVload->vgCfgVersion, pVload->replica, + pVgroup->pDb->dbCfgVersion, pVgroup->vgCfgVersion, pVgroup->numOfVnodes); mnodeSendAlterVgroupMsg(pVgroup); } } @@ -656,7 +661,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "onlineVnodes"); + strcpy(pSchema[cols].name, "onlines"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -671,13 +676,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -840,7 +845,8 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { SVnodeCfg *pCfg = &pVnode->cfg; pCfg->vgId = htonl(pVgroup->vgId); - pCfg->cfgVersion = htonl(pDb->cfgVersion); + pCfg->dbCfgVersion = htonl(pDb->dbCfgVersion); + pCfg->vgCfgVersion = htonl(pVgroup->vgCfgVersion); pCfg->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCfg->totalBlocks = htonl(pDb->cfg.totalBlocks); pCfg->maxTables = htonl(maxTables + 1); @@ -855,10 +861,12 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; pCfg->walLevel = pDb->cfg.walLevel; - pCfg->replications = (int8_t) pVgroup->numOfVnodes; + pCfg->vgReplica = (int8_t) pVgroup->numOfVnodes; pCfg->wals = 3; pCfg->quorum = pDb->cfg.quorum; pCfg->update = pDb->cfg.update; + pCfg->cacheLastRow = pDb->cfg.cacheLastRow; + pCfg->dbReplica = pDb->cfg.replications; SVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { @@ -978,6 +986,19 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); + + if (mnodeMsg->pBatchMasterMsg) { + ++mnodeMsg->pBatchMasterMsg->received; + if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received + >= mnodeMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(mnodeMsg); + + return; + } + dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { @@ -987,6 +1008,19 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { .pObj = pVgroup }; sdbDeleteRow(&row); + + if (mnodeMsg->pBatchMasterMsg) { + ++mnodeMsg->pBatchMasterMsg->received; + if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received + >= mnodeMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, mnodeMsg->code); + } + + mnodeDestroySubMsg(mnodeMsg); + + return; + } + dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); } } @@ -1174,3 +1208,14 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) { mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups); } + +int32_t mnodeGetVgidVer(int8_t *cver) { + int32_t iver = ((int32_t)cver[0]) * 10000 + ((int32_t)cver[1]) * 100 + (int32_t)cver[2]; + return iver; +} + +void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { + cver[0] = (int8_t)((int32_t)(iver % 1000000) / 10000); + cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); + cver[2] = (int8_t)(iver % 100); +} diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index 53981238a76ee3976689f77315eabab118816e81..c0699b05b364927492b8c2656bead0e14d46ab5a 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -50,7 +50,7 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); - mnodeGetMnodeEpSetForShell(epSet); + mnodeGetMnodeEpSetForShell(epSet, true); rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h index 0616006650eb2d53cdf7bd70b68c9f60748deac5..439e4cab72e4192d9c13a02519d268586a06f13e 100644 --- a/src/os/inc/osMemory.h +++ b/src/os/inc/osMemory.h @@ -52,7 +52,7 @@ void taosTMemset(void *ptr, int c); free((void *)(x)); \ x = 0; \ } \ - } while (0); + } while (0) #ifdef TAOS_MEM_CHECK #ifdef TAOS_MEM_CHECK_TEST diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index a71e74e97f4d9910414a5e801b89a1968d1df050..74e1bd487815942651111a2aa85e31650281bf20 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -29,12 +29,13 @@ extern "C" { #endif // TAOS_OS_FUNC_SEMPHONE_PTHREAD -bool taosCheckPthreadValid(pthread_t thread); -int64_t taosGetPthreadId(); -void taosResetPthread(pthread_t *thread); -bool taosComparePthread(pthread_t first, pthread_t second); +bool taosCheckPthreadValid(pthread_t thread); +int64_t taosGetSelfPthreadId(); +int64_t taosGetPthreadId(pthread_t thread); +void taosResetPthread(pthread_t* thread); +bool taosComparePthread(pthread_t first, pthread_t second); int32_t taosGetPId(); -int32_t taosGetCurrentAPPName(char *name, int32_t* len); +int32_t taosGetCurrentAPPName(char* name, int32_t* len); #ifdef __cplusplus } diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index baf7687dd03e1ee4f6dd92e3204244b3d31b7a1f..13d3fa407921ebf93992d3d0da5c8dabc41f5e44 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -59,6 +59,7 @@ extern "C" { // TAOS_OS_FUNC_SOCKET int32_t taosSetNonblocking(SOCKET sock, int32_t on); +void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT diff --git a/src/os/inc/osString.h b/src/os/inc/osString.h index b2846a31bccfc61865d5729d93d48a484e0eb9b7..e07bea4f40b0ec907acf915f976344c49cdaaae8 100644 --- a/src/os/inc/osString.h +++ b/src/os/inc/osString.h @@ -39,7 +39,7 @@ extern "C" { do { \ strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ - } while (0); + } while (0) #ifndef TAOS_OS_FUNC_STRING_STR2INT64 int64_t tsosStr2int64(char *str); diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 9eb8c18a40a11fac9ada037163011cdcf92201fb..d379e56ed83bd19e4b9ff5336e95cd019f296a36 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -31,7 +31,8 @@ int tsem_wait(tsem_t* sem) { #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } -int64_t taosGetPthreadId() { return (int64_t)pthread_self(); } +int64_t taosGetSelfPthreadId() { return (int64_t)pthread_self(); } +int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } void taosResetPthread(pthread_t *thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } int32_t taosGetPId() { return getpid(); } diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index c7c9d774271555ae5989aa18b7e00d325a0eddde..729471247f884977e65d86166cabf06641581e2f 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -39,6 +39,10 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() { + signal(SIGPIPE, SIG_IGN); +} + void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemphone.c index 1f723540f695e4a4609a9ec74e773a1559e7f30a..0bc760b35e9016f63a8cd40e1035891cf36e27ae 100644 --- a/src/os/src/windows/wSemphone.c +++ b/src/os/src/windows/wSemphone.c @@ -25,14 +25,16 @@ bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } void taosResetPthread(pthread_t *thread) { thread->p = 0; } -int64_t taosGetPthreadId() { +int64_t taosGetPthreadId(pthread_t thread) { #ifdef PTW32_VERSION - return pthread_getw32threadid_np(pthread_self()); + return pthread_getw32threadid_np(thread); #else - return (int64_t)pthread_self(); + return (int64_t)thread; #endif } +int64_t taosGetSelfPthreadId() { return taosGetPthreadId(pthread_self()); } + bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 3b091b269931e644d9f8c2c01ba3c9cb9ddc520c..9697c5e65fa374255c82596ec99d0c8b3e5b9aad 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -46,6 +46,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 755dad2d850c24192f5eee35c49492378ba8fb73..72604e79b710f09e67fde825915db63bc62243d9 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -50,9 +50,16 @@ static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char * char head[512] = {0}; char body[512] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[pContext->parser->httpVersion], - httpCode, httpCodeStr, httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[httpVersion], httpCode, + httpCodeStr, httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); @@ -136,7 +143,7 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { else httpCode = 400; - if (pContext->parser->httpCode != 0) { + if (pContext->parser && pContext->parser->httpCode != 0) { httpCode = pContext->parser->httpCode; } @@ -164,9 +171,16 @@ void httpSendSuccResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->parser->httpVersion], - httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[httpVersion], + httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); @@ -177,9 +191,16 @@ void httpSendOptionResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->parser->httpVersion], - httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[httpVersion], + httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); diff --git a/src/query/inc/qSyntaxtreefunction.h b/src/query/inc/qArithmeticOperator.h similarity index 91% rename from src/query/inc/qSyntaxtreefunction.h rename to src/query/inc/qArithmeticOperator.h index 6f91d2f7eded38425cf8414b6998e0162557b2c8..f13c63acc3509b13b1696c6360d4135e00079ba2 100644 --- a/src/query/inc/qSyntaxtreefunction.h +++ b/src/query/inc/qArithmeticOperator.h @@ -23,7 +23,7 @@ extern "C" { typedef void (*_bi_consumer_fn_t)(void *left, void *right, int32_t numOfLeft, int32_t numOfRight, void *output, int32_t order); -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr); +_bi_consumer_fn_t getArithmeticOperatorFn(int32_t leftType, int32_t rightType, int32_t optr); #ifdef __cplusplus } diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 28c1c7b838236d10a3f67fa7bbfc15a9b36c4612..39af7261efc222c8a6bcfc809288c256eccb0970 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -74,9 +74,7 @@ typedef struct tExprNode { }; } tExprNode; -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); - -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); tExprNode* exprTreeFromBinary(const void* data, size_t size); @@ -87,6 +85,8 @@ void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 9b29ad909a8eb81ea32442dd463102f0fb9460b8..32a68549fabd73104c56ea89a705f7cf5a6a30d1 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,13 +33,6 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -typedef struct SGroupResInfo { - int32_t groupId; - int32_t numOfDataPages; - int32_t pageId; - int32_t rowId; -} SGroupResInfo; - typedef struct SResultRowPool { int32_t elemSize; int32_t blockSize; @@ -72,6 +65,12 @@ typedef struct SResultRow { union {STimeWindow win; char* key;}; // start key of current time window } SResultRow; +typedef struct SGroupResInfo { + int32_t rowId; + int32_t index; + SArray* pRows; // SArray +} SGroupResInfo; + /** * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. @@ -89,7 +88,6 @@ typedef struct SResultRowInfo { int32_t size:24; // number of result set int32_t capacity; // max capacity int32_t curIndex; // current start active index - int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key } SResultRowInfo; @@ -185,14 +183,14 @@ typedef struct SQueryRuntimeEnv { uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; SResultRowInfo windowResInfo; - STSBuf* pTSBuf; + STSBuf* pTsBuf; STSCursor cur; SQueryCostInfo summary; void* pQueryHandle; void* pSecQueryHandle; // another thread for bool stableQuery; // super table query or not bool topBotQuery; // TODO used bitwise flag - bool groupbyNormalCol; // denote if this is a groupby normal column query + bool groupbyColumn; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not bool timeWindowInterpo;// if the time window start/end required interpolation bool queryWindowIdentical; // all query time windows are identical for all tables in one group @@ -206,6 +204,8 @@ typedef struct SQueryRuntimeEnv { int32_t* rowCellInfoOffset;// offset value for each row result cell info char** prevRow; char** nextRow; + + SArithmeticSupport *sasArray; } SQueryRuntimeEnv; enum { @@ -239,6 +239,7 @@ typedef struct SQInfo { int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context int64_t startExecTs; // start to exec timestamp + char* sql; // query sql string } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 442e61750b06d269ef48c5e74ede7646f7ac62e1..7742d151a06456a8b3c10e8b1f0aeba6203a28b7 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto); void tHistogramPrint(SHistogramInfo* pHisto); -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val); +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val); SHeapEntry* tHeapCreate(int32_t numOfEntries); void tHeapSort(SHeapEntry* pEntry, int32_t len); diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 513ab090f921b15ca18e384643ab79f51a119a8d..56e676ef167b8c9f1a694ef0571d1b0318ef6992 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -73,23 +73,27 @@ typedef struct SQuerySQL { SStrToken selectToken; // sql string } SQuerySQL; +typedef struct SCreatedTableInfo { + SStrToken name; // table name token + SStrToken stableName; // super table name token , for using clause + SArray *pTagVals; // create by using super table, tag value + char *fullname; // table full name + STagData tagdata; // true tag data, super table full name is in STagData + int8_t igExist; // ignore if exists +} SCreatedTableInfo; + typedef struct SCreateTableSQL { - struct SStrToken name; // meter name, create table [meterName] xxx - bool existCheck; - - int8_t type; // create normal table/from super table/ stream + SStrToken name; // table name, create table [name] xxx + int8_t type; // create normal table/from super table/ stream + bool existCheck; + struct { - SArray *pTagColumns; // SArray - SArray *pColumns; // SArray + SArray *pTagColumns; // SArray + SArray *pColumns; // SArray } colInfo; - - struct { - SStrToken stableName; // super table name, for using clause - SArray *pTagVals; // create by using metric, tag value - STagData tagdata; - } usingInfo; - - SQuerySQL *pSelect; + + SArray *childTableInfo; // SArray + SQuerySQL *pSelect; } SCreateTableSQL; typedef struct SAlterTableSQL { @@ -116,7 +120,8 @@ typedef struct SCreateDBInfo { int32_t compressionLevel; SStrToken precision; bool ignoreExists; - int8_t update; + int8_t update; + int8_t cachelast; SArray *keep; } SCreateDBInfo; @@ -198,16 +203,16 @@ typedef struct tSQLExpr { } tSQLExpr; // used in select clause. select from xxx -typedef struct tSQLExprItem { +typedef struct tSqlExprItem { tSQLExpr *pNode; // The list of expressions char * aliasName; // alias name, null-terminated string -} tSQLExprItem; +} tSqlExprItem; // todo refactor by using SArray typedef struct tSQLExprList { int32_t nExpr; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ - tSQLExprItem *a; /* One entry for each expression */ + tSqlExprItem *a; /* One entry for each expression */ } tSQLExprList; /** @@ -229,62 +234,63 @@ SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder); SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder); -tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType); +tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType); -void tSQLExprDestroy(tSQLExpr *); +void tSqlExprDestroy(tSQLExpr *pExpr); -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); +tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); -void tSQLExprListDestroy(tSQLExprList *pList); +void tSqlExprListDestroy(tSQLExprList *pList); -SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, +SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); -SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pMetricName, - SArray *pTagVals, SQuerySQL *pSelect, int32_t type); +SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type); -void tSQLExprNodeDestroy(tSQLExpr *pExpr); +void tSqlExprNodeDestroy(tSQLExpr *pExpr); -SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type); +SAlterTableSQL * tAlterTableSqlElems(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type); +SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists); void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); +void freeCreateTableInfo(void* p); -SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type); +SSqlInfo * setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists); -void SQLInfoDestroy(SSqlInfo *pInfo); +void SqlInfoDestroy(SSqlInfo *pInfo); void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); -void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); -void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip); -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); +void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); +void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip); +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); // prefix show db.tables; -void setDBName(SStrToken *pCpxName, SStrToken *pDB); +void setDbName(SStrToken *pCpxName, SStrToken *pDb); -tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optType); +tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType); -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); +tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); +void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken); +void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type); void *ParseAlloc(void *(*mallocProc)(size_t)); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index fb71c8a5fe0a7948c09c4bb4d83ed77b5d291389..4620e3d61e5e57c1072a7c3b3f6ca5face080d37 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -24,36 +24,31 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) + int32_t getOutputInterResultBufSize(SQuery* pQuery); -void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type); -void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); -SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); +size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv); +int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); +void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -int32_t initWindowResInfo(SResultRowInfo* pWindowResInfo, int32_t size, int16_t type); +void resetResultRowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); +int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); +void closeAllResultRows(SResultRowInfo* pResultRowInfo); -void cleanupTimeWindowInfo(SResultRowInfo* pWindowResInfo); -void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pWindowResInfo); -void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num); +int32_t initResultRow(SResultRow *pResultRow); +void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); +bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); +void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); -void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv); -int32_t numOfClosedTimeWindow(SResultRowInfo* pWindowResInfo); -void closeTimeWindow(SResultRowInfo* pWindowResInfo, int32_t slot); -void closeAllTimeWindow(SResultRowInfo* pWindowResInfo); -void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order); +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pWindowResInfo, int32_t slot) { - assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); - return pWindowResInfo->pResult[slot]; +static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { + assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); + return pResultRowInfo->pResult[slot]; } -#define curTimeWindowIndex(_winres) ((_winres)->curIndex) -#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) - -bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot); - -int32_t initResultRow(SResultRow *pResultRow); - static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); @@ -71,8 +66,6 @@ bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); -size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv); - SResultRowPool* initResultRowPool(size_t size); SResultRow* getNewResultRow(SResultRowPool* p); int64_t getResultRowPoolMemSize(SResultRowPool* p); @@ -80,7 +73,6 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); bool isPointInterpoQuery(SQuery *pQuery); diff --git a/src/query/inc/queryLog.h b/src/query/inc/queryLog.h index a1c447a6eb6a5f14d3b5913203a9412acce3b5c4..d4e909d33a5f79ee11de5e13b21ada2930a738bf 100644 --- a/src/query/inc/queryLog.h +++ b/src/query/inc/queryLog.h @@ -23,14 +23,14 @@ extern "C" { #include "tlog.h" extern int32_t qDebugFlag; -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; -#define qFatal(...) { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", 255, __VA_ARGS__); }} -#define qError(...) { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", 255, __VA_ARGS__); }} -#define qWarn(...) { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", 255, __VA_ARGS__); }} -#define qInfo(...) { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", 255, __VA_ARGS__); }} -#define qDebug(...) { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} -#define qTrace(...) { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", 255, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", 255, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", 255, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", 255, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index e2b3bb6cbffc21bac7462284963fda72880b9906..dda15fb508d17bfb80ba95d2d9249fc0021d4ae8 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -112,29 +112,29 @@ cmd ::= SHOW dbPrefix(X) STABLES. { cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } //drop configure for tables cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { X.n += Z.n; - setDropDBTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y); + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y); } -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDBTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y); } +cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y); } cmd ::= DROP DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } cmd ::= DROP USER ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } cmd ::= DROP ACCOUNT ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } @@ -149,16 +149,16 @@ cmd ::= DESCRIBE ids(X) cpxName(Y). { } /////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// -cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } -cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} +cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } +cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} -cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} -cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} +cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} +cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. @@ -168,47 +168,47 @@ ids(A) ::= ID(X). {A = X; } ids(A) ::= STRING(X). {A = X; } %type ifexists {SStrToken} -ifexists(X) ::= IF EXISTS. {X.n = 1;} -ifexists(X) ::= . {X.n = 0;} +ifexists(X) ::= IF EXISTS. { X.n = 1;} +ifexists(X) ::= . { X.n = 0;} %type ifnotexists {SStrToken} -ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} -ifnotexists(X) ::= . {X.n = 0;} +ifnotexists(X) ::= IF NOT EXISTS. { X.n = 1;} +ifnotexists(X) ::= . { X.n = 0;} /////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// //create option for dnode/db/user/account cmd ::= CREATE DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). - { setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} + { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSQL(pInfo, &X, &Y);} +cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} -pps(Y) ::= . {Y.n = 0; } -pps(Y) ::= PPS INTEGER(X). {Y = X; } +pps(Y) ::= . { Y.n = 0; } +pps(Y) ::= PPS INTEGER(X). { Y = X; } -tseries(Y) ::= . {Y.n = 0; } -tseries(Y) ::= TSERIES INTEGER(X). {Y = X; } +tseries(Y) ::= . { Y.n = 0; } +tseries(Y) ::= TSERIES INTEGER(X). { Y = X; } -dbs(Y) ::= . {Y.n = 0; } -dbs(Y) ::= DBS INTEGER(X). {Y = X; } +dbs(Y) ::= . { Y.n = 0; } +dbs(Y) ::= DBS INTEGER(X). { Y = X; } -streams(Y) ::= . {Y.n = 0; } -streams(Y) ::= STREAMS INTEGER(X). {Y = X; } +streams(Y) ::= . { Y.n = 0; } +streams(Y) ::= STREAMS INTEGER(X). { Y = X; } -storage(Y) ::= . {Y.n = 0; } -storage(Y) ::= STORAGE INTEGER(X). {Y = X; } +storage(Y) ::= . { Y.n = 0; } +storage(Y) ::= STORAGE INTEGER(X). { Y = X; } -qtime(Y) ::= . {Y.n = 0; } -qtime(Y) ::= QTIME INTEGER(X). {Y = X; } +qtime(Y) ::= . { Y.n = 0; } +qtime(Y) ::= QTIME INTEGER(X). { Y = X; } -users(Y) ::= . {Y.n = 0; } -users(Y) ::= USERS INTEGER(X). {Y = X; } +users(Y) ::= . { Y.n = 0; } +users(Y) ::= USERS INTEGER(X). { Y = X; } -conns(Y) ::= . {Y.n = 0; } -conns(Y) ::= CONNS INTEGER(X). {Y = X; } +conns(Y) ::= . { Y.n = 0; } +conns(Y) ::= CONNS INTEGER(X). { Y = X; } -state(Y) ::= . {Y.n = 0; } -state(Y) ::= STATE ids(X). {Y = X; } +state(Y) ::= . { Y.n = 0; } +state(Y) ::= STATE ids(X). { Y = X; } %type acct_optr {SCreateAcctSQL} acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { @@ -240,6 +240,7 @@ fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; } update(Y) ::= UPDATE INTEGER(X). { Y = X; } +cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } %type db_optr {SCreateDBInfo} db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} @@ -258,6 +259,7 @@ db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strto db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } %type alter_db_optr {SCreateDBInfo} alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} @@ -269,23 +271,24 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { X.type = 0; - tSQLSetColumnType (&A, &X); + tSqlSetColumnType (&A, &X); } //define binary type, e.g., binary(10), nchar(10) typename(A) ::= ids(X) LP signed(Y) RP. { - if (Y <= 0) { - X.type = 0; - tSQLSetColumnType(&A, &X); - } else { - X.type = -Y; // negative value of name length - tSQLSetColumnType(&A, &X); - } + if (Y <= 0) { + X.type = 0; + tSqlSetColumnType(&A, &X); + } else { + X.type = -Y; // negative value of name length + tSqlSetColumnType(&A, &X); + } } %type signed {int64_t} @@ -294,36 +297,60 @@ signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} ////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE ifnotexists(Y) ids(X) cpxName(Z) create_table_args. { - X.n += Z.n; - setCreatedTableName(pInfo, &X, &Y); +cmd ::= CREATE TABLE create_table_args. {} +cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} + +%type create_table_list{SCreateTableSQL*} +%destructor create_table_list{destroyCreateTableSql($$);} +create_table_list(A) ::= create_from_stable(Z). { + SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); + pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); + + taosArrayPush(pCreateTable->childTableInfo, &Z); + pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; + A = pCreateTable; +} + +create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { + taosArrayPush(X->childTableInfo, &Z); + A = X; } %type create_table_args{SCreateTableSQL*} -create_table_args(A) ::= LP columnlist(X) RP. { - A = tSetCreateSQLElems(X, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { + A = tSetCreateSqlElems(X, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + + V.n += Z.n; + setCreatedTableName(pInfo, &V, &U); } // create super table -create_table_args(A) ::= LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateSQLElems(X, Y, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { + A = tSetCreateSqlElems(X, Y, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + + V.n += Z.n; + setCreatedTableName(pInfo, &V, &U); } // create table by using super table // create table table_name using super_table_name tags(tag_values1, tag_values2) -create_table_args(A) ::= USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. { - X.n += F.n; - A = tSetCreateSQLElems(NULL, NULL, &X, Y, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +%type create_from_stable{SCreatedTableInfo} +create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. { + X.n += F.n; + V.n += Z.n; + A = createNewChildTableInfo(&X, Y, &V, &U); } // create stream // create table table_name as select count(*) from super_table_name interval(time) -create_table_args(A) ::= AS select(S). { - A = tSetCreateSQLElems(NULL, NULL, NULL, NULL, S, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); +create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { + A = tSetCreateSqlElems(NULL, NULL, S, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + + V.n += Z.n; + setCreatedTableName(pInfo, &V, &U); } %type column{TAOS_FIELD} @@ -335,7 +362,7 @@ columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(T // The information used for a column is the name and type of column: // tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) column(A) ::= ids(X) typename(Y). { - tSQLSetColumnInfo(&A, &X, &Y); + tSqlSetColumnInfo(&A, &X, &Y); } %type tagitemlist {SArray*} @@ -345,10 +372,10 @@ column(A) ::= ids(X) typename(Y). { tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1); } tagitemlist(A) ::= tagitem(X). { A = tVariantListAppend(NULL, &X, -1); } -tagitem(A) ::= INTEGER(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } -tagitem(A) ::= FLOAT(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } -tagitem(A) ::= STRING(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } -tagitem(A) ::= BOOL(X). {toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= INTEGER(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= FLOAT(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= STRING(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } +tagitem(A) ::= BOOL(X). { toTSDBType(X.type); tVariantCreate(&A, &X); } tagitem(A) ::= NULL(X). { X.type = 0; tVariantCreate(&A, &X); } tagitem(A) ::= MINUS(X) INTEGER(Y).{ @@ -383,7 +410,7 @@ tagitem(A) ::= PLUS(X) FLOAT(Y). { %type select {SQuerySQL*} %destructor select {doDestroyQuerySql($$);} select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { - A = tSetQuerySQLElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); + A = tSetQuerySqlElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); } %type union {SSubclauseInfo*} @@ -394,33 +421,33 @@ union(Y) ::= LP union(X) RP. { Y = X; } union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); } union(Y) ::= union(Z) UNION ALL LP select(X) RP. { Y = appendSelectClause(Z, X); } -cmd ::= union(X). { setSQLInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } +cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } // Support for the SQL exprssion without from & where subclauses, e.g., // select current_database(), // select server_version(), select client_version(), // select server_state(); select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySQLElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + A = tSetQuerySqlElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } // selcollist is a list of expressions that are to become the return // values of the SELECT statement. The "*" in statements like // "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. %type selcollist {tSQLExprList*} -%destructor selcollist {tSQLExprListDestroy($$);} +%destructor selcollist {tSqlExprListDestroy($$);} %type sclp {tSQLExprList*} -%destructor sclp {tSQLExprListDestroy($$);} +%destructor sclp {tSqlExprListDestroy($$);} sclp(A) ::= selcollist(X) COMMA. {A = X;} sclp(A) ::= . {A = 0;} selcollist(A) ::= sclp(P) expr(X) as(Y). { - A = tSQLExprListAppend(P, X, Y.n?&Y:0); + A = tSqlExprListAppend(P, X, Y.n?&Y:0); } selcollist(A) ::= sclp(P) STAR. { - tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - A = tSQLExprListAppend(P, pNode, 0); + tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); + A = tSqlExprListAppend(P, pNode, 0); } // An option "AS " phrase that can follow one of the expressions that @@ -445,11 +472,11 @@ tablelist(A) ::= ids(X) cpxName(Y). { } tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { - toTSDBType(X.type); - toTSDBType(Z.type); - X.n += Y.n; - A = tVariantListAppendToken(NULL, &X, -1); - A = tVariantListAppendToken(A, &Z, -1); + toTSDBType(X.type); + toTSDBType(Z.type); + X.n += Y.n; + A = tVariantListAppendToken(NULL, &X, -1); + A = tVariantListAppendToken(A, &Z, -1); } tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { @@ -460,11 +487,11 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { } tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { - toTSDBType(X.type); - toTSDBType(F.type); - X.n += Z.n; - A = tVariantListAppendToken(Y, &X, -1); - A = tVariantListAppendToken(A, &F, -1); + toTSDBType(X.type); + toTSDBType(F.type); + X.n += Z.n; + A = tVariantListAppendToken(Y, &X, -1); + A = tVariantListAppendToken(A, &F, -1); } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" @@ -526,9 +553,9 @@ item(A) ::= ids(X) cpxName(Y). { } %type sortorder {int} -sortorder(A) ::= ASC. {A = TSDB_ORDER_ASC; } -sortorder(A) ::= DESC. {A = TSDB_ORDER_DESC;} -sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend order +sortorder(A) ::= ASC. { A = TSDB_ORDER_ASC; } +sortorder(A) ::= DESC. { A = TSDB_ORDER_DESC;} +sortorder(A) ::= . { A = TSDB_ORDER_ASC; } // Ascending order by default //group by clause %type groupby_opt {SArray*} @@ -536,8 +563,8 @@ sortorder(A) ::= . {A = TSDB_ORDER_ASC;} //default is descend orde %type grouplist {SArray*} %destructor grouplist {taosArrayDestroy($$);} -groupby_opt(A) ::= . {A = 0;} -groupby_opt(A) ::= GROUP BY grouplist(X). {A = X;} +groupby_opt(A) ::= . { A = 0;} +groupby_opt(A) ::= GROUP BY grouplist(X). { A = X;} grouplist(A) ::= grouplist(X) COMMA item(Y). { A = tVariantListAppend(X, &Y, -1); @@ -549,7 +576,7 @@ grouplist(A) ::= item(X). { //having clause, ignore the input condition in having %type having_opt {tSQLExpr*} -%destructor having_opt {tSQLExprDestroy($$);} +%destructor having_opt {tSqlExprDestroy($$);} having_opt(A) ::=. {A = 0;} having_opt(A) ::= HAVING expr(X). {A = X;} @@ -571,7 +598,7 @@ slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). {A.limit = Y; A.offset = X;} %type where_opt {tSQLExpr*} -%destructor where_opt {tSQLExprDestroy($$);} +%destructor where_opt {tSqlExprDestroy($$);} where_opt(A) ::= . {A = 0;} where_opt(A) ::= WHERE expr(X). {A = X;} @@ -579,67 +606,67 @@ where_opt(A) ::= WHERE expr(X). {A = X;} /////////////////////////// Expression Processing ///////////////////////////// // %type expr {tSQLExpr*} -%destructor expr {tSQLExprDestroy($$);} +%destructor expr {tSqlExprDestroy($$);} expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->token.z = X.z; A->token.n = (Z.z - X.z + 1);} -expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);} -expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);} +expr(A) ::= ID(X). { A = tSqlExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ALL);} + +expr(A) ::= INTEGER(X). { A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= FLOAT(X). { A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= STRING(X). { A = tSqlExprIdValueCreate(&X, TK_STRING);} +expr(A) ::= NOW(X). { A = tSqlExprIdValueCreate(&X, TK_NOW); } +expr(A) ::= VARIABLE(X). { A = tSqlExprIdValueCreate(&X, TK_VARIABLE);} +expr(A) ::= BOOL(X). { A = tSqlExprIdValueCreate(&X, TK_BOOL);} // ordinary functions: min(x), max(x), top(k, 20) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSQLExprCreateFunction(Y, &X, &E, X.type); } +expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); } // for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); } +expr(A) ::= ID(X) LP STAR RP(Y). { A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); } // is (not) null expression -expr(A) ::= expr(X) IS NULL. {A = tSQLExprCreate(X, NULL, TK_ISNULL);} -expr(A) ::= expr(X) IS NOT NULL. {A = tSQLExprCreate(X, NULL, TK_NOTNULL);} +expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);} +expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);} // relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);} +expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);} +expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);} +expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);} +expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} +expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} +expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} -expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); } +expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} +expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } // binary arithmetic expression -expr(A) ::= expr(X) PLUS expr(Y). {A = tSQLExprCreate(X, Y, TK_PLUS); } -expr(A) ::= expr(X) MINUS expr(Y). {A = tSQLExprCreate(X, Y, TK_MINUS); } -expr(A) ::= expr(X) STAR expr(Y). {A = tSQLExprCreate(X, Y, TK_STAR); } -expr(A) ::= expr(X) SLASH expr(Y). {A = tSQLExprCreate(X, Y, TK_DIVIDE);} -expr(A) ::= expr(X) REM expr(Y). {A = tSQLExprCreate(X, Y, TK_REM); } +expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); } +expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); } +expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); } +expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);} +expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } // like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); } +expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } //in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, TK_IN); } +expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSQLExpr*)Y, TK_IN); } %type exprlist {tSQLExprList*} -%destructor exprlist {tSQLExprListDestroy($$);} +%destructor exprlist {tSqlExprListDestroy($$);} %type expritem {tSQLExpr*} -%destructor expritem {tSQLExprDestroy($$);} +%destructor expritem {tSqlExprDestroy($$);} -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSQLExprListAppend(X,Y,0);} -exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);} +exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0);} +exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0);} expritem(A) ::= expr(X). {A = X;} expritem(A) ::= . {A = 0;} @@ -649,8 +676,8 @@ cmd ::= RESET QUERY CACHE. { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} ///////////////////////////////////ALTER TABLE statement////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { X.n += F.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { @@ -659,15 +686,15 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { toTSDBType(A.type); SArray* K = tVariantListAppendToken(NULL, &A, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } //////////////////////////////////ALTER TAGS statement///////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { X.n += Y.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { X.n += Z.n; @@ -675,8 +702,8 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { toTSDBType(Y.type); SArray* A = tVariantListAppendToken(NULL, &Y, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { @@ -688,8 +715,8 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { toTSDBType(Z.type); A = tVariantListAppendToken(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { @@ -699,14 +726,14 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { SArray* A = tVariantListAppendToken(NULL, &Y, -1); A = tVariantListAppend(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } ////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION INTEGER(Y). {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} -cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &X);} +cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} +cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);} +cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);} %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 38bb0b8a71cbc4d219fd39c209a812660dd0a1b7..51048bbe72368cb6869092b65d5f035f26f3545b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -112,11 +112,10 @@ extern "C" { #define TOP_BOTTOM_QUERY_LIMIT 100 enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - FIRST_STAGE_MERGE = 0x10u, - SECONDARY_STAGE_MERGE = 0x20u, + MASTER_SCAN = 0x0u, + REVERSE_SCAN = 0x1u, + REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan + MERGE_STAGE = 0x20u, }; #define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) @@ -191,8 +190,8 @@ typedef struct SQLFunctionCtx { int64_t nStartQueryTimestamp; // timestamp range of current query when function is executed on a specific data block int32_t numOfParams; tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */ - int64_t * ptsList; // corresponding timestamp array list - void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + int64_t *ptsList; // corresponding timestamp array list + void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; tVariant tag; @@ -215,18 +214,12 @@ typedef struct SQLAggFuncElem { void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version - // some sql function require scan data twice or more, e.g.,stddev + // some sql function require scan data twice or more, e.g.,stddev, percentile void (*xNextStep)(SQLFunctionCtx *pCtx); - /* - * finalizer must be called after all xFunction has been executed to - * generated final result. Otherwise, the value in aOutputBuf is a intern result. - */ + // finalizer must be called after all xFunction has been executed to generated final result. void (*xFinalize)(SQLFunctionCtx *pCtx); - - void (*distMergeFunc)(SQLFunctionCtx *pCtx); - - void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx); + void (*mergeFunc)(SQLFunctionCtx *pCtx); int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; diff --git a/src/client/src/tscFunctionImpl.c b/src/query/src/qAggMain.c similarity index 87% rename from src/client/src/tscFunctionImpl.c rename to src/query/src/qAggMain.c index 62e55c033f53b53aaf9035b2c7196ffe7faef522..2997d56326b664488a1942df0150343bb5b0f6f9 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/query/src/qAggMain.c @@ -15,27 +15,26 @@ #include "os.h" #include "qAst.h" -#include "qExtbuffer.h" #include "qFill.h" #include "qHistogram.h" #include "qPercentile.h" -#include "qSyntaxtreefunction.h" #include "qTsbuf.h" #include "taosdef.h" #include "taosmsg.h" #include "tscLog.h" #include "tscSubquery.h" -#include "tscompression.h" #include "tsqlfunction.h" -#include "tutil.h" #include "ttype.h" -#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) -#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes) +#define GET_INPUT_DATA_LIST(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) +#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) + +#define GET_TS_LIST(x) ((TSKEY*)&((x)->ptsList[(x)->startOffset])) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) #define GET_TRUE_DATA_TYPE() \ int32_t type = 0; \ - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { \ + if (pCtx->currentStage == MERGE_STAGE) { \ type = pCtx->outputType; \ assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ } else { \ @@ -48,7 +47,7 @@ break; \ } \ GET_RES_INFO(ctx)->numOfRes = (res); \ - } while (0); + } while (0) #define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); @@ -62,7 +61,7 @@ } \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ } \ - } while (0); + } while (0) #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ @@ -395,7 +394,7 @@ static void count_function(SQLFunctionCtx *pCtx) { } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_CHAR_INDEX(pCtx, i); + char *val = GET_INPUT_DATA(pCtx, i); if (isNull(val, pCtx->inputType)) { continue; } @@ -420,7 +419,7 @@ static void count_function(SQLFunctionCtx *pCtx) { } static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -434,7 +433,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void count_func_merge(SQLFunctionCtx *pCtx) { - int64_t *pData = (int64_t *)GET_INPUT_CHAR(pCtx); + int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->aOutputBuf) += pData[i]; } @@ -482,24 +481,23 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId DO_UPDATE_TAG_COLUMNS(ctx, k); \ (num) += 1; \ } \ - } while (0); + } while (0) #define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ -do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ - (num) += 1; \ - } \ - } while (0); - + (num) += 1; \ + } \ + } while (0) #define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ for (int32_t i = 0; i < ((ctx)->size); ++i) { \ if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ continue; \ } \ - TSKEY key = (ctx)->ptsList[i]; \ + TSKEY key = GET_TS_DATA(ctx, i); \ UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ } @@ -526,7 +524,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { *retVal += GET_DOUBLE_VAL((const char*)&(pCtx->preAggVals.statis.sum)); } } else { // computing based on the true data block - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { @@ -559,7 +557,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { } static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -609,21 +607,21 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { +static void sum_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); assert(pCtx->stableQuery); - + for (int32_t i = 0; i < pCtx->size; ++i) { - char * input = GET_INPUT_CHAR_INDEX(pCtx, i); + char * input = GET_INPUT_DATA(pCtx, i); SSumInfo *pInput = (SSumInfo *)input; if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + notNullElems++; - + switch (type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -638,25 +636,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { } } } - - return notNullElems; -} - -static void sum_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - - SET_VAL(pCtx, notNullElems, 1); - SSumInfo *pSumInfo = (SSumInfo *)pCtx->aOutputBuf; - - if (notNullElems > 0) { - // pCtx->numOfIteratedElems += notNullElems; - pSumInfo->hasResult = DATA_SET_FLAG; - } -} -static void sum_func_second_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - SET_VAL(pCtx, notNullElems, 1); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -709,15 +689,14 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY en return BLK_DATA_ALL_NEEDED; } - return BLK_DATA_ALL_NEEDED; - // TODO pCtx->aOutputBuf is the previous windowRes output buffer, not current unloaded block. so the following filter - // is invalid -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { // data in current block is not earlier than current result -// return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->aOutputBuf is + // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { // data in current block is not earlier than current result + return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { @@ -730,16 +709,14 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end return BLK_DATA_ALL_NEEDED; } - return BLK_DATA_ALL_NEEDED; - // TODO pCtx->aOutputBuf is the previous windowRes output buffer, not current unloaded block. so the following filter - // is invalid - -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { -// return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->aOutputBuf is + // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { + return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } ////////////////////////////////////////////////////////////////////////////////////////////// @@ -768,7 +745,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.sum)); } } else { - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); @@ -803,7 +780,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -843,33 +820,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current buffer is null - continue; - } - - pAvgInfo->sum += pInput->sum; - pAvgInfo->num += pInput->num; - } - - // if the data set hasResult is not set, the result is null - if (pAvgInfo->num > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_second_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->aOutputBuf; - char * input = GET_INPUT_CHAR(pCtx); + char * input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; @@ -890,7 +843,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { static void avg_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { @@ -924,7 +877,15 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (pCtx->preAggVals.isSet) { *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(*notNullElems >= 0); + + if (*notNullElems == 0) { + return; + } + if (*notNullElems == 0){ + return; + } + void * tval = NULL; int16_t index = 0; @@ -947,7 +908,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (index < 0 || index >= pCtx->size + pCtx->startOffset) { index = 0; } - + + // the index is the original position, not the relative position key = pCtx->ptsList[index]; } @@ -998,7 +960,9 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, return; } - void *p = GET_INPUT_CHAR(pCtx); + void *p = GET_INPUT_DATA_LIST(pCtx); + TSKEY *tsList = GET_TS_LIST(pCtx); + *notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { @@ -1017,7 +981,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; - TSKEY k = pCtx->ptsList[i]; + TSKEY k = tsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); } @@ -1148,7 +1112,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_CHAR_INDEX(pCtx, i); + char *input = GET_INPUT_DATA(pCtx, i); if (input[bytes] != DATA_SET_FLAG) { continue; } @@ -1202,17 +1166,6 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp } static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void min_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1224,16 +1177,6 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { } static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); - - SET_VAL(pCtx, numOfElems, 1); - if (numOfElems > 0) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void max_func_second_merge(SQLFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1245,8 +1188,8 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { } static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - TSKEY key = pCtx->ptsList[index]; + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY key = GET_TS_DATA(pCtx, index); int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -1285,7 +1228,7 @@ static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin } static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1301,7 +1244,7 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1334,7 +1277,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { double *retVal = &pStd->res; double avg = pStd->avg; - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { @@ -1385,7 +1328,7 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { avg_function_f(pCtx, index); } else { double avg = pStd->avg; - void * pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void * pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; @@ -1491,14 +1434,14 @@ static void first_function(SQLFunctionCtx *pCtx) { // handle the null value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - TSKEY k = pCtx->ptsList[i]; + TSKEY k = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, k); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1517,7 +1460,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1525,7 +1468,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1534,7 +1477,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); @@ -1565,7 +1508,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { // find the first not null value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -1583,7 +1526,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { } static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1598,26 +1541,9 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - -static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); - char * pData = GET_INPUT_CHAR(pCtx); + char * pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; @@ -1652,7 +1578,7 @@ static void last_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; @@ -1660,7 +1586,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[i]; + TSKEY ts = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1675,7 +1601,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1689,7 +1615,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -1697,7 +1623,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->complete = true; // set query completed } else { // in case of ascending order check, all data needs to be checked SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { @@ -1711,7 +1637,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); @@ -1745,7 +1671,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; @@ -1769,7 +1695,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1787,32 +1713,13 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); } -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - // the input data is null - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - /* * in the secondary merge(local reduce), the output is limited by the * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); +static void last_dist_func_merge(SQLFunctionCtx *pCtx) { + char *pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1841,7 +1748,7 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { */ static void last_row_function(SQLFunctionCtx *pCtx) { assert(pCtx->size >= 1); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); // assign the last element in current data block assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); @@ -1852,12 +1759,13 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // set the result to final result buffer in case of super table query if (pCtx->stableQuery) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; + pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } else { - DO_UPDATE_TAG_COLUMNS(pCtx, pCtx->ptsList[pCtx->size - 1]); + TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); + DO_UPDATE_TAG_COLUMNS(pCtx, ts); } SET_VAL(pCtx, pCtx->size, 1); @@ -1888,7 +1796,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 dst->timestamp = tsKey; int32_t size = 0; - if (stage == SECONDARY_STAGE_MERGE || stage == FIRST_STAGE_MERGE) { + if (stage == MERGE_STAGE) { memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { @@ -1909,7 +1817,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 (dst)->timestamp = (src)->timestamp; \ (dst)->v = (src)->v; \ memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ - } while (0); + } while (0) static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { @@ -2151,7 +2059,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { return (STopBotInfo*) pCtx->aOutputBuf; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer return GET_ROWCELL_INTERBUF(pResInfo); @@ -2222,7 +2130,8 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { tmp += POINTER_BYTES * pCtx->param[0].i64Key; size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - +// assert(pCtx->param[0].i64Key > 0); + for (int32_t i = 0; i < pCtx->param[0].i64Key; ++i) { pTopBotInfo->res[i] = (tValuePair*) tmp; pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); @@ -2248,13 +2157,15 @@ static void top_function(SQLFunctionCtx *pCtx) { assert(pRes->num >= 0); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } notNullElems++; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2271,7 +2182,7 @@ static void top_function(SQLFunctionCtx *pCtx) { } static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2280,36 +2191,16 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { assert(pRes->num >= 0); SET_VAL(pCtx, 1, 1); - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + TSKEY ts = GET_TS_DATA(pCtx, index); + + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void top_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void top_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2337,14 +2228,15 @@ static void bottom_function(SQLFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } notNullElems++; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2361,43 +2253,23 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY ts = GET_TS_DATA(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } STopBotInfo *pRes = getTopBotOutputInfo(pCtx); SET_VAL(pCtx, 1, 1); - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, - NULL, 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void bottom_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2493,7 +2365,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); } else { for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2518,7 +2390,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { // the second stage, calculate the true percentile value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2532,7 +2404,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { } static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2600,14 +2472,23 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////// +static void buildHistogramInfo(SAPercentileInfo* pInfo) { + pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); + pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); +} + static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return (SAPercentileInfo*) pCtx->aOutputBuf; + SAPercentileInfo* pInfo = NULL; + + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { + pInfo = (SAPercentileInfo*) pCtx->aOutputBuf; } else { - return GET_ROWCELL_INTERBUF(pResInfo); + pInfo = GET_ROWCELL_INTERBUF(pResInfo); } + + buildHistogramInfo(pInfo); + return pInfo; } static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { @@ -2627,9 +2508,11 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); + + assert(pInfo->pHisto->elems != NULL); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2653,7 +2536,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2671,42 +2554,7 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - size_t size = sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); //(SAPercentileInfo *)pCtx->aOutputBuf; - SHistogramInfo * pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, size); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - tHistogramDestroy(&pRes); - } - - SET_VAL(pCtx, 1, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); @@ -2741,7 +2589,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null assert(pOutput->pHisto->numOfElems > 0); @@ -2809,7 +2657,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { double(*param)[3] = pInfo->mat; double x = pInfo->startVal; - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); int32_t numOfElem = 0; switch (pCtx->inputType) { @@ -2869,7 +2717,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { } static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2884,7 +2732,7 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { int32_t *p = pData; LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_TINYINT: { int8_t *p = pData; LEASTSQR_CAL(param, pInfo->startVal, p, 0, pCtx->param[1].dKey); @@ -2971,7 +2819,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { @@ -2996,7 +2844,7 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { } INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pCtx->aOutputBuf += pCtx->inputBytes; @@ -3050,7 +2898,7 @@ static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } @@ -3070,7 +2918,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { // TODO difference in date column static void diff_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_CHAR(pCtx); + void *data = GET_INPUT_DATA_LIST(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); int32_t notNullElems = 0; @@ -3078,8 +2926,9 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - TSKEY * pTimestamp = pCtx->ptsOutputBuf; - + TSKEY* pTimestamp = pCtx->ptsOutputBuf; + TSKEY* tsList = GET_TS_LIST(pCtx); + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *pData = (int32_t *)data; @@ -3095,13 +2944,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); // direct previous may be null - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3127,13 +2976,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3159,12 +3008,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } @@ -3189,13 +3038,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3222,12 +3071,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3253,13 +3102,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3300,12 +3149,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { } else { \ *(type *)(ctx)->aOutputBuf = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64Key)); \ *(type *)(&(ctx)->param[1].i64Key) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = (ctx)->ptsList[index]; \ + *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ } \ } while (0); static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3325,7 +3174,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { } else { *(int32_t *)pCtx->aOutputBuf = *(int32_t *)pData - (int32_t)pCtx->param[1].i64Key; pCtx->param[1].i64Key = *(int32_t *)pData; - *(int64_t *)pCtx->ptsOutputBuf = pCtx->ptsList[index]; + *(int64_t *)pCtx->ptsOutputBuf = GET_TS_DATA(pCtx, index); } break; }; @@ -3378,7 +3227,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; pCtx->param[1].pz = NULL; @@ -3389,7 +3238,7 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes; } @@ -3420,7 +3269,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { pCtx->param[0].dKey = DBL_MAX; pCtx->param[3].dKey = -DBL_MAX; } else { @@ -3469,7 +3318,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { goto _spread_over; } - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); numOfElems = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -3505,7 +3354,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { } static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3547,45 +3396,12 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -void spread_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t notNullElems = 0; - for (int32_t i = 0; i < pCtx->size; ++i) { - SSpreadInfo *input = (SSpreadInfo *)GET_INPUT_CHAR_INDEX(pCtx, i); - - /* no assign tag, the value is null */ - if (input->hasResult != DATA_SET_FLAG) { - continue; - } - - if (pResData->min > input->min) { - pResData->min = input->min; - } - - if (pResData->max < input->max) { - pResData->max = input->max; - } - - pResData->hasResult = DATA_SET_FLAG; - notNullElems++; - } - - if (notNullElems > 0) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - /* * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer */ void spread_func_sec_merge(SQLFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_CHAR(pCtx); + SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); if (pData->hasResult != DATA_SET_FLAG) { return; } @@ -3608,7 +3424,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { */ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (pResInfo->hasResult != DATA_SET_FLAG) { @@ -3663,26 +3479,25 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *tsList = GET_TS_LIST(pCtx); int32_t i = index; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); SPoint1* last = &pInfo->p; if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); + assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); assert(last->key == INT64_MIN); - last->key = primaryKey[tsIndex + i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); pInfo->dOutput += twa_get_area(pCtx->start, *last); @@ -3691,8 +3506,8 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t notNullElems++; i += step; } else if (pInfo->p.key == INT64_MIN) { - last->key = primaryKey[tsIndex + i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = last->key; @@ -3703,78 +3518,78 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t // calculate the value of switch(pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = (double) val[i]}; + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0); + float *val = (float*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0); + double *val = (double*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -3794,7 +3609,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t } static void twa_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_CHAR(pCtx); + void *data = GET_INPUT_DATA_LIST(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3802,11 +3617,15 @@ static void twa_function(SQLFunctionCtx *pCtx) { // skip null value int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { + while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { i += step; } - int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size); + int32_t notNullElems = 0; + if (i >= 0 && i < pCtx->size) { + notNullElems = twa_function_impl(pCtx, i, pCtx->size); + } + SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3818,135 +3637,15 @@ static void twa_function(SQLFunctionCtx *pCtx) { } } -//TODO refactor static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - + int32_t notNullElems = twa_function_impl(pCtx, index, 1); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - int32_t i = pCtx->startOffset; - int32_t size = pCtx->size; - - if (pCtx->start.key != INT64_MIN) { - assert(pInfo->p.key == INT64_MIN); - - pInfo->p.key = primaryKey[index]; - GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, pInfo->p); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += 1; - } else if (pInfo->p.key == INT64_MIN) { - pInfo->p.key = primaryKey[index]; - GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->p.key; - notNullElems++; - i += 1; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = (double) val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);//((pInfo->p.val + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->p.key); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3958,34 +3657,6 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -static void twa_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; - char * indicator = pCtx->aInputElemBuf; - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { - STwaInfo *pInput = (STwaInfo*) indicator; - - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } - - numOfNotNull++; - pBuf->dOutput += pInput->dOutput; - - pBuf->win = pInput->win; - pBuf->p = pInput->p; - } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; - } -} - /* * To copy the input to interResBuf to avoid the input buffer space be over writen * by next input data. The TWA function only applies to each table, so no merge procedure @@ -4028,8 +3699,10 @@ static void interp_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + assert(pCtx->startOffset == 0); + if (pCtx->size == 1) { - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } else { /* @@ -4055,13 +3728,13 @@ static void interp_function(SQLFunctionCtx *pCtx) { } else if (pInfo->type == TSDB_FILL_SET_VALUE) { tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType, true); } else if (pInfo->type == TSDB_FILL_PREV) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, 0); + char *data = GET_INPUT_DATA(pCtx, 0); assignVal(pCtx->aOutputBuf, data, pCtx->outputBytes, pCtx->outputType); SET_VAL(pCtx, pCtx->size, 1); } else if (pInfo->type == TSDB_FILL_LINEAR) { - char *data1 = GET_INPUT_CHAR_INDEX(pCtx, 0); - char *data2 = GET_INPUT_CHAR_INDEX(pCtx, 1); + char *data1 = GET_INPUT_DATA(pCtx, 0); + char *data2 = GET_INPUT_DATA(pCtx, 1); TSKEY key1 = pCtx->ptsList[0]; TSKEY key2 = pCtx->ptsList[1]; @@ -4123,14 +3796,14 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - const char *input = GET_INPUT_CHAR(pCtx); + const char *input = GET_INPUT_DATA_LIST(pCtx); // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_CHAR_INDEX(pCtx, i); + char *d = GET_INPUT_DATA(pCtx, i); tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -4140,7 +3813,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { } static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4202,7 +3875,6 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { return resultVal; } - static bool rate_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; @@ -4224,18 +3896,17 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return true; } - static void rate_function(SQLFunctionCtx *pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + int32_t notNullElems = 0; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { tscDebug("%p rate_function() index of null data:%d", pCtx, i); continue; @@ -4283,7 +3954,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { } static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4291,8 +3962,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // NOTE: keep the intermediate result into the interResultBuf SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - + TSKEY *primaryKey = GET_TS_LIST(pCtx); + int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4324,42 +3995,6 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } - - -static void rate_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - tscDebug("rate_func_merge() size:%d", pCtx->size); - - SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; - char *indicator = pCtx->aInputElemBuf; - - assert(1 == pCtx->size); - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(SRateInfo)) { - SRateInfo *pInput = (SRateInfo *)indicator; - if (DATA_SET_FLAG != pInput->hasResult) { - continue; - } - - numOfNotNull++; - memcpy(pBuf, pInput, sizeof(SRateInfo)); - tscDebug("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, - pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); - } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; - } - - return; -} - - - static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -4368,12 +4003,10 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; - tscDebug("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + tscDebug("%p rate_func_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } - - static void rate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4397,14 +4030,13 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } - static void irate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - + TSKEY *primaryKey = GET_TS_LIST(pCtx); + tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); if (pCtx->size < 1) { @@ -4412,7 +4044,7 @@ static void irate_function(SQLFunctionCtx *pCtx) { } for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { tscDebug("%p irate_function() index of null data:%d", pCtx, i); continue; @@ -4455,15 +4087,15 @@ static void irate_function(SQLFunctionCtx *pCtx) { } static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4493,7 +4125,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); + char * input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SRateInfo *pInput = (SRateInfo *)input; @@ -4525,11 +4157,6 @@ static void sumrate_func_merge(SQLFunctionCtx *pCtx) { do_sumrate_merge(pCtx); } -static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { - tscDebug("%p sumrate_func_second_merge() process ...", pCtx); - do_sumrate_merge(pCtx); -} - static void sumrate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4593,7 +4220,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, count_func_merge, - count_func_merge, count_load_data_info, }, { @@ -4608,7 +4234,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, sum_func_merge, - sum_func_second_merge, statisRequired, }, { @@ -4623,7 +4248,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, avg_finalizer, avg_func_merge, - avg_func_second_merge, statisRequired, }, { @@ -4638,7 +4262,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, min_func_merge, - min_func_second_merge, statisRequired, }, { @@ -4653,7 +4276,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, max_func_merge, - max_func_second_merge, statisRequired, }, { @@ -4668,7 +4290,6 @@ SQLAggFuncElem aAggs[] = {{ stddev_next_step, stddev_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4683,7 +4304,6 @@ SQLAggFuncElem aAggs[] = {{ percentile_next_step, percentile_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4698,7 +4318,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, apercentile_finalizer, apercentile_func_merge, - apercentile_func_second_merge, dataBlockRequired, }, { @@ -4713,7 +4332,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, firstFuncRequired, }, { @@ -4728,7 +4346,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, lastFuncRequired, }, { @@ -4743,8 +4360,7 @@ SQLAggFuncElem aAggs[] = {{ noop2, no_next_step, last_row_finalizer, - noop1, - last_dist_func_second_merge, + last_dist_func_merge, dataBlockRequired, }, { @@ -4760,7 +4376,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, top_func_merge, - top_func_second_merge, dataBlockRequired, }, { @@ -4776,7 +4391,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, bottom_func_merge, - bottom_func_second_merge, dataBlockRequired, }, { @@ -4790,7 +4404,6 @@ SQLAggFuncElem aAggs[] = {{ spread_function_f, no_next_step, spread_function_finalizer, - spread_func_merge, spread_func_sec_merge, count_load_data_info, }, @@ -4805,7 +4418,6 @@ SQLAggFuncElem aAggs[] = {{ twa_function_f, no_next_step, twa_function_finalizer, - twa_func_merge, twa_function_copy, dataBlockRequired, }, @@ -4821,7 +4433,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, leastsquares_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4836,7 +4447,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4851,7 +4461,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4866,7 +4475,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4881,7 +4489,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, ts_comp_finalize, copy_function, - copy_function, dataBlockRequired, }, { @@ -4896,7 +4503,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4911,7 +4517,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4926,7 +4531,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4941,7 +4545,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4956,7 +4559,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, noop1, - noop1, dataBlockRequired, }, // distributed version used in two-stage aggregation processes @@ -4972,7 +4574,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, first_dist_func_merge, - first_dist_func_second_merge, firstDistFuncRequired, }, { @@ -4987,7 +4588,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, last_dist_func_merge, - last_dist_func_second_merge, lastDistFuncRequired, }, { @@ -5001,7 +4601,6 @@ SQLAggFuncElem aAggs[] = {{ do_sum_f, // todo filter handle no_next_step, doFinalizer, - noop1, copy_function, dataBlockRequired, }, @@ -5016,7 +4615,6 @@ SQLAggFuncElem aAggs[] = {{ rate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, @@ -5031,7 +4629,6 @@ SQLAggFuncElem aAggs[] = {{ irate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, @@ -5047,7 +4644,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5062,7 +4658,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5077,7 +4672,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5092,7 +4686,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5107,6 +4700,5 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, noop1, noop1, - noop1, dataBlockRequired, }}; diff --git a/src/query/src/qSyntaxtreefunction.c b/src/query/src/qArithmeticOperator.c similarity index 99% rename from src/query/src/qSyntaxtreefunction.c rename to src/query/src/qArithmeticOperator.c index 7f7fca2c1e0ca6ce85f0df1c1b6e6dce82d8c000..0955d48df0bacbdf748c8a06f6a6364387d9be9d 100644 --- a/src/query/src/qSyntaxtreefunction.c +++ b/src/query/src/qArithmeticOperator.c @@ -15,7 +15,7 @@ #include "os.h" -#include "qSyntaxtreefunction.h" +#include "qArithmeticOperator.h" #include "taosdef.h" #include "tutil.h" @@ -1234,7 +1234,7 @@ _bi_consumer_fn_t rem_function_arraylist[8][10] = { //////////////////////////////////////////////////////////////////////////////////////////////////////////// -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr) { +_bi_consumer_fn_t getArithmeticOperatorFn(int32_t leftType, int32_t rightType, int32_t optr) { switch (optr) { case TSDB_BINARY_OP_ADD: return add_function_arraylist[leftType][rightType]; diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index a65f4a6dc95a586896372a648dc19ad2bb88b154..c23a79419660f0e474854e005db86a18971c0c96 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -16,29 +16,18 @@ #include "os.h" #include "exception.h" +#include "qArithmeticOperator.h" #include "qAst.h" -#include "qSyntaxtreefunction.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" #include "tname.h" +#include "tschemautil.h" #include "tsdb.h" #include "tskiplist.h" #include "tsqlfunction.h" -#include "tstoken.h" -#include "tschemautil.h" - -typedef struct { - char* v; - int32_t optr; -} SEndPoint; - -typedef struct { - SEndPoint* start; - SEndPoint* end; -} SQueryCond; static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { if (pLeft->nodeType == TSQL_NODE_COL) { @@ -53,323 +42,6 @@ static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, co } } -void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { - if (pNode == NULL) { - return; - } - - if (pNode->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&pNode, fp); - } else if (pNode->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy(pNode->pVal); - } else if (pNode->nodeType == TSQL_NODE_COL) { - free(pNode->pSchema); - } - - free(pNode); -} - -void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { - if (*pExpr == NULL) { - return; - } - - if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); - tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); - - if (fp != NULL) { - fp((*pExpr)->_node.info); - } - } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy((*pExpr)->pVal); - free((*pExpr)->pVal); - } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { - free((*pExpr)->pSchema); - } - - free(*pExpr); - *pExpr = NULL; -} - -// todo check for malloc failure -static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { - int32_t optr = queryColInfo->optr; - - if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || - optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(SEndPoint)); - pCond->start->optr = queryColInfo->optr; - pCond->start->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(SEndPoint)); - pCond->end->optr = queryColInfo->optr; - pCond->end->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - -static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { - SSkipListIterator* iter = NULL; - - SQueryCond cond = {0}; - if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { - //todo handle error - } - - if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); - } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); - } - - if (cond.start != NULL) { - int32_t optr = cond.start->optr; - - if (optr == TSDB_RELATION_EQUAL) { // equals - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - if (ret != 0) { - break; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal - bool comp = true; - int32_t ret = 0; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - assert(ret >= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_GREATER) { - continue; - } else { - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; - } - } - } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal - bool comp = true; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - tSkipListDestroyIter(iter); - - comp = true; - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - } else { - assert(0); - } - } else { - int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; - if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - bool comp = true; - int32_t ret = 0; - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); - assert(ret <= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_LESS) { - continue; - } else { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; // no need to compare anymore - } - } - } else { - assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); - if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || - (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } - } - } - - free(cond.start); - free(cond.end); - tSkipListDestroyIter(iter); -} - -static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - //non-leaf nodes, recursively traverse the expression tree in the post-root order - if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { - if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - if (filterItem(pLeft, pItem, param)) { - return true; - } - - // left child does not satisfy the query condition, try right child - return filterItem(pRight, pItem, param); - } else { // and - if (!filterItem(pLeft, pItem, param)) { - return false; - } - - return filterItem(pRight, pItem, param); - } - } - - // handle the leaf node - param->setupInfoFn(pExpr, param->pExtInfo); - return param->nodeFilterFn(pItem, pExpr->_node.info); -} - -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { - taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); - } - } - tSkipListDestroyIter(iter); -} - -static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - bool addToResult = false; - - SSkipListNode *pNode = tSkipListIterGet(iter); - char * pData = SL_GET_NODE_DATA(pNode); - - tstr *name = (tstr*) tsdbGetTableName((void*) pData); - - // todo speed up by using hash - if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { - if (pQueryInfo->optr == TSDB_RELATION_IN) { - addToResult = pQueryInfo->compare(name, pQueryInfo->q); - } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { - addToResult = !pQueryInfo->compare(name, pQueryInfo->q); - } - } else { - addToResult = filterFp(pNode, pQueryInfo); - } - - if (addToResult) { - STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(res, &info); - } - } - - tSkipListDestroyIter(iter); -} - -// post-root order traverse syntax tree -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { - if (pExpr == NULL) { - return; - } - - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - // column project - if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { - assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); - - param->setupInfoFn(pExpr, param->pExtInfo); - - tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { - tQueryIndexColumn(pSkipList, pQueryInfo, result); - } else { - tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); - } - - return; - } - - // The value of hasPK is always 0. - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; - assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); - - //apply the hierarchical expression to every node in skiplist for find the qualified nodes - tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - -#if 0 - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } - - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); - - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); -#endif -} - static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: { @@ -430,7 +102,73 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf } } -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { + if (pNode == NULL) { + return; + } + + if (pNode->nodeType == TSQL_NODE_EXPR) { + tExprTreeDestroy(&pNode, fp); + } else if (pNode->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy(pNode->pVal); + } else if (pNode->nodeType == TSQL_NODE_COL) { + free(pNode->pSchema); + } + + free(pNode); +} + +void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { + if (*pExpr == NULL) { + return; + } + + if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { + tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); + tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); + + if (fp != NULL) { + fp((*pExpr)->_node.info); + } + } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy((*pExpr)->pVal); + free((*pExpr)->pVal); + } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { + free((*pExpr)->pSchema); + } + + free(*pExpr); + *pExpr = NULL; +} + +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + //non-leaf nodes, recursively traverse the expression tree in the post-root order + if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { + if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (exprTreeApplayFilter(pLeft, pItem, param)) { + return true; + } + + // left child does not satisfy the query condition, try right child + return exprTreeApplayFilter(pRight, pItem, param); + } else { // and + if (!exprTreeApplayFilter(pLeft, pItem, param)) { + return false; + } + + return exprTreeApplayFilter(pRight, pItem, param); + } + } + + // handle the leaf node + param->setupInfoFn(pExpr, param->pExtInfo); + return param->nodeFilterFn(pItem, pExpr->_node.info); +} + +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { return; @@ -442,7 +180,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, /* the left output has result from the left child syntax tree */ char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } /* the right output has result from the right child syntax tree */ @@ -450,7 +188,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, char *pdata = malloc(sizeof(int64_t) * numOfRows); if (pRight->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } if (pLeft->nodeType == TSQL_NODE_EXPR) { @@ -459,11 +197,11 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, * exprLeft + exprRight * the type of returned value of one expression is always double float precious */ - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); // set input buffer char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); @@ -475,14 +213,14 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); } } else if (pLeft->nodeType == TSQL_NODE_COL) { // column data specified on left-hand-side char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId); if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); @@ -494,12 +232,12 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); // both columns are descending order, do not reverse the source data fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); @@ -511,13 +249,13 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } else { // column data specified on left-hand-side if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows); @@ -527,7 +265,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, TSDB_ORDER_ASC); } } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e83cafa0ac526b88546f7a29a144e3cdc86cbe50..be4d849fb99e1f030f48f08e43e7407cd3676451 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -35,7 +35,7 @@ * check if the primary column is load by default, otherwise, the program will * forced to load primary column explicitly. */ -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -53,7 +53,7 @@ #define TIME_WINDOW_COPY(_dst, _src) do {\ (_dst).skey = (_src).skey;\ (_dst).ekey = (_src).ekey;\ -} while (0); +} while (0) enum { // when query starts to execute, this status will set @@ -86,7 +86,6 @@ typedef struct { int32_t status; // query status TSKEY lastKey; // the lastKey value before query executed STimeWindow w; // whole query time window - STimeWindow curWindow; // current query window int32_t windowIndex; // index of active time window result for interval query STSCursor cur; } SQueryStatusInfo; @@ -178,11 +177,10 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { #define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) // todo move to utility -static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); +static int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo); static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); static void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); -static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, @@ -195,7 +193,6 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); @@ -291,7 +288,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static int32_t getMergeResultGroupId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getMergeResultGroupId(int32_t groupIndex) { int32_t base = 50000000; return base + (groupIndex * 10000); } @@ -464,60 +461,84 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, char *pData, +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { + bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - int32_t *p1 = - (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (p1 != NULL) { - pWindowResInfo->curIndex = *p1; + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; + } + + if (p1 != NULL) { + for(int32_t i = pResultRowInfo->size - 1; i >= 0; --i) { + if (pResultRowInfo->pResult[i] == (*p1)) { + pResultRowInfo->curIndex = i; + existed = true; + break; + } + } + } } else { - if (!masterscan) { // not master scan, do not add new timewindow - return NULL; + if (p1 != NULL) { // group by column query + return *p1; } + } + if (!existed) { // TODO refactor // more than the capacity, reallocate the resources - if (pWindowResInfo->size >= pWindowResInfo->capacity) { + if (pResultRowInfo->size >= pResultRowInfo->capacity) { int64_t newCapacity = 0; - if (pWindowResInfo->capacity > 10000) { - newCapacity = (int64_t)(pWindowResInfo->capacity * 1.25); + if (pResultRowInfo->capacity > 10000) { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); } else { - newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5); + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); } - char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pWindowResInfo->pResult = (SResultRow **)t; + pResultRowInfo->pResult = (SResultRow **)t; - int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc); + int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; + memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); - pWindowResInfo->capacity = (int32_t)newCapacity; + pResultRowInfo->capacity = (int32_t)newCapacity; } - SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool); - pWindowResInfo->pResult[pWindowResInfo->size] = pResult; - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + SResultRow *pResult = NULL; + + if (p1 == NULL) { + pResult = getNewResultRow(pRuntimeEnv->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + } else { + pResult = *p1; } - // add a new result set for a new group - pWindowResInfo->curIndex = pWindowResInfo->size++; - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), - (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + pResultRowInfo->pResult[pResultRowInfo->size] = pResult; + pResultRowInfo->curIndex = pResultRowInfo->size++; } // too many time window in query - if (pWindowResInfo->size > MAX_INTERVAL_TIME_WINDOW) { + if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return getResultRow(pWindowResInfo, pWindowResInfo->curIndex); + return getResultRow(pResultRowInfo, pResultRowInfo->curIndex); } // get the correct time window according to the handled timestamp @@ -591,7 +612,6 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf if (pData->num >= numOfRowsPerPage) { // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer @@ -614,26 +634,20 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, - STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) { +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, + bool masterscan, SResultRow** pResult, int64_t groupId) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - // todo refactor - int64_t uid = getResultInfoUId(pRuntimeEnv); - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, uid); + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); if (pResultRow == NULL) { - *newWind = false; - - // no master scan, no result generated means error occurs - return masterscan? -1:0; + *pResult = NULL; + return TSDB_CODE_SUCCESS; } - *newWind = true; - // not assign result buffer yet, add new result buffer if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -703,81 +717,60 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -static int32_t updateResultRowCurrentIndex(SResultRowInfo* pWindowResInfo, TSKEY lastKey, bool ascQuery) { - int32_t i = 0; +static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; - - int32_t numOfClosed = 0; - for (i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pResult = pWindowResInfo->pResult[i]; + int32_t i = 0; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; if (pResult->closed) { - numOfClosed += 1; - continue; + break; } - TSKEY ekey = pResult->win.ekey; - if ((ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { - closeTimeWindow(pWindowResInfo, i); + // new closed result rows + if (timeWindowInterpo) { + if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { + if (i > 0) { // the first time window, the startInterp is false. + assert(pResult->startInterp); + } + + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } } else { - skey = pResult->win.skey; - break; + if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } } } - // all windows are closed, set the last one to be the skey + // all result rows are closed, set the last one to be the skey if (skey == TSKEY_INITIAL_VAL) { - assert(i == pWindowResInfo->size); - pWindowResInfo->curIndex = pWindowResInfo->size - 1; + pResultRowInfo->curIndex = pResultRowInfo->size - 1; } else { - pWindowResInfo->curIndex = i; - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; - } - - return numOfClosed; -} - -/** - * NOTE: the query status only set for the first scan of master scan. - */ -static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->scanFlag != MASTER_SCAN || pWindowResInfo->size == 0) { - return pWindowResInfo->size; - } - - // no qualified results exist, abort check - int32_t numOfClosed = 0; - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - // query completed - if ((lastKey >= pQuery->current->win.ekey && ascQuery) || (lastKey <= pQuery->current->win.ekey && (!ascQuery))) { - closeAllTimeWindow(pWindowResInfo); - - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); - } else { // set the current index to be the last unclosed window - numOfClosed = updateResultRowCurrentIndex(pWindowResInfo, lastKey, ascQuery); - - // the number of completed slots are larger than the threshold, return current generated results to client. - if (numOfClosed > pQuery->rec.threshold) { - qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", - GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - } else { - qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, - numOfClosed); + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } } - } - // output has reached the limitation, set query completed - if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && - pRuntimeEnv->scanFlag == MASTER_SCAN) { - setQueryStatus(pQuery, QUERY_COMPLETED); + pResultRowInfo->curIndex = i + 1; // current not closed result object + pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; } +} - assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); - return numOfClosed; +static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery, bool timeWindowInterpo) { + if ((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey && (!ascQuery))) { + closeAllResultRows(pResultRowInfo); + pResultRowInfo->curIndex = pResultRowInfo->size - 1; + } else { + int32_t step = ascQuery? 1:-1; + doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey - step, ascQuery, timeWindowInterpo); + } } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, @@ -820,52 +813,44 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, - int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { - SQuery * pQuery = pRuntimeEnv->pQuery; +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { + SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool hasPrev = pCtx[0].preAggVals.isSet; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; - pCtx[k].size = forwardStep; - pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); - - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; - } + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; + pCtx[k].size = forwardStep; + pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } + int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunction(&pCtx[k]); - } + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { + pCtx[k].preAggVals.isSet = false; + } - // restore it - pCtx[k].preAggVals.isSet = hasPrev; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); } + + // restore it + pCtx[k].preAggVals.isSet = hasPrev; } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { - SQuery * pQuery = pRuntimeEnv->pQuery; +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset) { + SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); } } } @@ -977,8 +962,7 @@ static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { return NULL; } -static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, - SArray *pDataBlock) { +static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { if (pDataBlock == NULL) { return NULL; } @@ -989,15 +973,9 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas int32_t functionId = pQuery->pExpr1[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - - sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); - sas->colList = pQuery->colList; - sas->numOfCols = pQuery->numOfCols; - sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); - - if (sas->data == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); + sas->colList = pQuery->colList; + sas->numOfCols = pQuery->numOfCols; // here the pQuery->colList and sas->colList are identical int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); @@ -1100,13 +1078,13 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32 return true; } -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) { +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, + int32_t rowIndex) { if (pDataBlock == NULL) { return; } SQuery* pQuery = pRuntimeEnv->pQuery; - int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); @@ -1126,6 +1104,45 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY return ts; } +static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray *pDataBlock, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + if (!pRuntimeEnv->timeWindowInterpo) { + return; + } + + assert(pDataBlock != NULL); + + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); + + TSKEY *tsCols = (TSKEY *)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } + + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + } +} + /** * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions * @param pRuntimeEnv @@ -1141,22 +1158,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + TSKEY *tsCols = NULL; if (pDataBlock != NULL) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); tsCols = (TSKEY *)(pColInfo->pData); } - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -1166,80 +1180,50 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { - tfree(sasArray); - return; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + goto _end; } int32_t forwardStep = 0; int32_t startPos = pQuery->pos; - // in case of repeat scan/reverse scan, no new time window added. - if (hasTimeWindow) { - TSKEY ekey = reviseWindowEkey(pQuery, &win); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - - // prev time window not interpolation yet. - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { - for(int32_t j = prevIndex; j < curIndex; ++j) { - SResultRow *pRes = pWindowResInfo->pResult[j]; - - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + TSKEY ekey = reviseWindowEkey(pQuery, &win); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows); + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { + for(int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow *pRes = pWindowResInfo->pResult[j]; + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; } - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - assert (ret == TSDB_CODE_SUCCESS); // null data, too many state code - } + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &w, masterScan, &pResult, groupId); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - // window start key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = pQuery->pos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } + int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step; - - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - } + doBlockwiseApplyFunctions(pRuntimeEnv, &w, startPos, 0, tsCols, pDataBlockInfo->rows); } - bool pStatus = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + assert (ret == TSDB_CODE_SUCCESS); } - int32_t index = pWindowResInfo->curIndex; - STimeWindow nextWin = win; + // window start key interpolation + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &win, pQuery->pos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos); @@ -1248,50 +1232,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (!hasTimeWindow) { - continue; - } - - TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); + ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); // window start(end) key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } - - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1)*step; - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - } - } - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &nextWin, startPos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } - pWindowResInfo->curIndex = index; } else { /* * the sqlfunctionCtx parameters should be set done before all functions are invoked, @@ -1307,28 +1260,14 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } } + _end: if (pRuntimeEnv->timeWindowInterpo) { - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock); + int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, rowIndex); } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); - } - - tfree(sasArray); } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - if (isNull(pData, type)) { // ignore the null value - return -1; - } - - int32_t GROUPRESULTID = 1; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; // not assign result buffer yet, add new result buffer, TODO remove it @@ -1340,25 +1279,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); - - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - uint64_t uid = groupIndex; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); - if (pResultRow == NULL) { return -1; } - int64_t v = -1; - switch(type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(pData); break; - case TSDB_DATA_TYPE_SMALLINT: v = GET_INT16_VAL(pData); break; - case TSDB_DATA_TYPE_INT: v = GET_INT32_VAL(pData); break; - case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break; - } + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, groupIndex); + assert (pResultRow != NULL); + int64_t v = -1; + GET_TYPED_DATA(v, int64_t, type, pData); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (pResultRow->key == NULL) { pResultRow->key = malloc(varDataTLen(pData)); @@ -1372,7 +1300,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; } @@ -1427,7 +1355,7 @@ static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQuery *pQuery = pRuntimeEnv->pQuery; - STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first @@ -1439,8 +1367,8 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { #if defined(_DEBUG_VIEW) printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag.i64Key, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder, - pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex); + elem.ts, key, elem.tag.i64Key, pQuery->order.order, pRuntimeEnv->pTsBuf->tsOrder, + pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); #endif if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -1507,7 +1435,7 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]); + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); } else { GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } @@ -1567,22 +1495,19 @@ static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pData } static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* item = pQuery->current; + int64_t groupId = item->groupIndex; + SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0); TSKEY *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL; - bool groupbyColumnValue = pRuntimeEnv->groupbyNormalCol; - - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + bool groupbyColumnValue = pRuntimeEnv->groupbyColumn; int16_t type = 0; int16_t bytes = 0; @@ -1594,8 +1519,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); pCtx[k].size = 1; } @@ -1610,9 +1535,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // from top to bottom in desc // from bottom to top in asc order - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { qDebug("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, - pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); + pQuery->order.order, pRuntimeEnv->pTsBuf->cur.order); } int32_t offset = -1; @@ -1622,7 +1547,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { offset = GET_COL_DATA_POS(pQuery, j, step); - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { int32_t ret = doTSJoinFilter(pRuntimeEnv, offset); if (ret == TS_JOIN_TAG_NOT_EQUALS) { break; @@ -1640,19 +1565,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); - int64_t ts = tsCols[offset]; + int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; - } - - if (!hasTimeWindow) { - continue; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { // null data, too many state code + goto _end; } // window start key interpolation @@ -1662,19 +1582,20 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { for (int32_t k = prevWindowIndex; k < curIndex; ++k) { SResultRow *pRes = pWindowResInfo->pResult[k]; + if (pRes->closed) { + assert(resultRowInterpolated(pResult, RESULT_ROW_START_INTERP) && resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + continue; + } - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &pRes->win, masterScan, &pResult, groupId); assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &pRes->win, offset); } // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, - &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1683,12 +1604,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); } - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &win, offset); + int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; - int32_t index = pWindowResInfo->curIndex; - while (1) { getNextTimeWindow(pQuery, &nextWin); if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || @@ -1701,27 +1620,28 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (hasTimeWindow) { - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); - closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); - } + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); + doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset); } + // restore the index, add the result row will move the index pWindowResInfo->curIndex = index; } else { // other queries // decide which group this rows belongs to according to current state value if (groupbyColumnValue) { char *val = groupbyColumnData + bytes * offset; + if (isNull(val, type)) { // ignore the null value + continue; + } int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } } @@ -1736,64 +1656,56 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS prevTs = tsCols[offset]; prevRowIndex = offset; - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { // if timestamp filter list is empty, quit current query - if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { setQueryStatus(pQuery, QUERY_COMPLETED); break; } } } - assert(offset >= 0); - if (tsCols != NULL) { - item->lastKey = tsCols[offset] + step; - } else { - item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step; + _end: + assert(offset >= 0 && tsCols != NULL); + if (prevTs != INT64_MIN) { + assert(prevRowIndex >= 0); + item->lastKey = prevTs + step; } - if (pRuntimeEnv->pTSBuf != NULL) { - item->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + // In case of all rows in current block are not qualified + if (pRuntimeEnv->timeWindowInterpo && prevRowIndex != -1) { + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, prevRowIndex); } - // todo refactor: extract method - for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); + if (pRuntimeEnv->pTsBuf != NULL) { + item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); } - - free(sasArray); } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQInfo = pQuery->current; - SResultRowInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + STableQueryInfo* pTableQueryInfo = pQuery->current; + SResultRowInfo* pResultRowInfo = &pRuntimeEnv->windowResInfo; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } - // update the lastkey of current table + // update the lastkey of current table for projection/aggregation query TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // interval query with limit applied int32_t numOfRes = 0; - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); - } else if (pRuntimeEnv->groupbyNormalCol) { - closeAllTimeWindow(pWindowResInfo); - numOfRes = pWindowResInfo->size; + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { + numOfRes = pResultRowInfo->size; + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); } else { // projection query - numOfRes = (int32_t)getNumOfResult(pRuntimeEnv); + numOfRes = (int32_t) getNumOfResult(pRuntimeEnv); // update the number of output result if (numOfRes > 0 && pQuery->checkBuffer == 1) { @@ -1808,8 +1720,8 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl setQueryStatus(pQuery, QUERY_COMPLETED); } - if (((pTableQInfo->lastKey > pTableQInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pTableQInfo->lastKey < pTableQInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { + if (((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { setQueryStatus(pQuery, QUERY_COMPLETED); } } @@ -1847,7 +1759,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = &tsCol[pCtx->startOffset]; + pCtx->ptsList = tsCol; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { @@ -1965,8 +1877,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); pRuntimeEnv->offset = calloc(pQuery->numOfOutput, sizeof(int16_t)); pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->sasArray = calloc(pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { + if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL || pRuntimeEnv->sasArray == NULL) { goto _clean; } @@ -2042,18 +1955,24 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->param[1].i64Key = pQuery->order.orderColId; } + if (functionId == TSDB_FUNC_ARITHM) { + pRuntimeEnv->sasArray[i].data = calloc(pQuery->numOfCols, POINTER_BYTES); + if (pRuntimeEnv->sasArray[i].data == NULL) { + goto _clean; + } + } + if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes; } - } *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; // if it is group by normal column, do not set output buffer, the output buffer is pResult // fixed output query/multi-output query for normal table - if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!pRuntimeEnv->groupbyColumn && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { resetDefaultResInfoOutputBuf(pRuntimeEnv); } @@ -2068,6 +1987,7 @@ _clean: tfree(pRuntimeEnv->pCtx); tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->rowCellInfoOffset); + tfree(pRuntimeEnv->sasArray); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2094,7 +2014,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); qDebug("QInfo:%p teardown runtime env", pQInfo); - cleanupTimeWindowInfo(&pRuntimeEnv->windowResInfo); + cleanupResultRowInfo(&pRuntimeEnv->windowResInfo); if (pRuntimeEnv->pCtx != NULL) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -2111,12 +2031,20 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->pCtx); } + if (pRuntimeEnv->sasArray != NULL) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + tfree(pRuntimeEnv->sasArray[i].data); + } + + tfree(pRuntimeEnv->sasArray); + } + pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pQInfo); - pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); + pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->keyBuf); @@ -2163,7 +2091,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { } // Note:top/bottom query is fixed output query - if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyColumn) { return true; } @@ -2191,8 +2119,8 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { // todo refactor with isLastRowQuery bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionID = pQuery->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_INTERP) { + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_INTERP) { return true; } } @@ -2624,11 +2552,14 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { } int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; - *status = BLK_DATA_NO_NEEDED; + SQueryCostInfo* pCost = &pRuntimeEnv->summary; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf > 0) { + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) { *status = BLK_DATA_ALL_NEEDED; } else { // check if this data block is required to load @@ -2642,16 +2573,13 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - bool hasTimeWindow = false; SResultRow* pResult = NULL; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; - STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } } @@ -2672,35 +2600,34 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW if ((*status) == BLK_DATA_NO_NEEDED) { qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pRuntimeEnv->summary.discardBlocks += 1; + pCost->discardBlocks += 1; } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); - - pRuntimeEnv->summary.loadBlockStatis += 1; + pCost->loadBlockStatis += 1; if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; + pCost->totalCheckedRows += pBlockInfo->rows; } } else { assert((*status) == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter - pRuntimeEnv->summary.loadBlockStatis += 1; + pCost->loadBlockStatis += 1; tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied - pRuntimeEnv->summary.discardBlocks += 1; + pCost->discardBlocks += 1; qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); (*status) = BLK_DATA_DISCARD; } - pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; - pRuntimeEnv->summary.loadBlocks += 1; + pCost->totalCheckedRows += pBlockInfo->rows; + pCost->loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); if (*pDataBlock == NULL) { return terrno; @@ -2758,7 +2685,7 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } numOfRows = lastPos - firstPos + 1; - midPos = (numOfRows >> 1) + firstPos; + midPos = (numOfRows >> 1u) + firstPos; if (key < keyList[midPos]) { lastPos = midPos - 1; @@ -2805,7 +2732,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { + if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyColumn && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { @@ -2850,13 +2777,9 @@ static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo if (QUERY_IS_ASC_QUERY(pQuery)) { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } @@ -2926,13 +2849,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_COMPLETED); } - if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - closeAllTimeWindow(&pRuntimeEnv->windowResInfo); - pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window - } else { - assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - } + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + closeAllResultRows(&pRuntimeEnv->windowResInfo); + pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window } return 0; @@ -3016,7 +2935,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set the join tag for first column SSqlFuncMsg *pFuncMsg = &pExprInfo->base; - if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTSBuf != NULL && + if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTsBuf != NULL && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { assert(pFuncMsg->numOfParams == 1); @@ -3037,50 +2956,6 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { } } -static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (!mergeFlag) { - pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; - pCtx[i].currentStage = FIRST_STAGE_MERGE; - - RESET_RESULT_INFO(pCtx[i].resultInfo); - aAggs[functionId].init(&pCtx[i]); - } - - pCtx[i].hasNull = true; - pCtx[i].nStartQueryTimestamp = timestamp; - pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); - - // in case of tag column, the tag information should be extracted from input buffer - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { - tVariantDestroy(&pCtx[i].tag); - - int32_t type = pCtx[i].outputType; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); - } else { - tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); - } - - } - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY) { - continue; - } - - aAggs[functionId].distMergeFunc(&pCtx[i]); - } -} - static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { switch (srcDataType) { @@ -3186,19 +3061,18 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim typedef struct SCompSupporter { STableQueryInfo **pTableQueryInfo; - int32_t * position; - SQInfo * pQInfo; + int32_t *rowIndex; + int32_t order; } SCompSupporter; int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { - int32_t left = *(int32_t *)pLeft; + int32_t left = *(int32_t *)pLeft; int32_t right = *(int32_t *)pRight; SCompSupporter * supporter = (SCompSupporter *)param; - SQueryRuntimeEnv *pRuntimeEnv = &supporter->pQInfo->runtimeEnv; - int32_t leftPos = supporter->position[left]; - int32_t rightPos = supporter->position[right]; + int32_t leftPos = supporter->rowIndex[left]; + int32_t rightPos = supporter->rowIndex[right]; /* left source is exhausted */ if (leftPos == -1) { @@ -3210,53 +3084,55 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return -1; } - SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; - SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); + STableQueryInfo** pList = supporter->pTableQueryInfo; - char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); - TSKEY leftTimestamp = GET_INT64_VAL(b1); + SResultRowInfo *pWindowResInfo1 = &(pList[left]->windowResInfo); + SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); + TSKEY leftTimestamp = pWindowRes1->win.skey; - SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; + SResultRowInfo *pWindowResInfo2 = &(pList[right]->windowResInfo); SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); - - char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); - TSKEY rightTimestamp = GET_INT64_VAL(b2); + TSKEY rightTimestamp = pWindowRes2->win.skey; if (leftTimestamp == rightTimestamp) { return 0; } - return leftTimestamp > rightTimestamp ? 1 : -1; + if (supporter->order == TSDB_ORDER_ASC) { + return (leftTimestamp > rightTimestamp)? 1:-1; + } else { + return (leftTimestamp < rightTimestamp)? 1:-1; + } } -int32_t mergeIntoGroupResult(SQInfo *pQInfo) { +int32_t mergeGroupResult(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); - int32_t ret = TSDB_CODE_SUCCESS; - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); while (pQInfo->groupIndex < numOfGroups) { SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); - ret = mergeIntoGroupResultImpl(pQInfo, group); - if (ret < 0) { // not enough disk space to save the data into disk - return -1; - } - pQInfo->groupIndex += 1; + int32_t ret = mergeIntoGroupResultImpl(pGroupResInfo, group, pQInfo); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } // this group generates at least one result, return results - if (ret > 0) { + pQInfo->groupIndex += 1; + if (taosArrayGetSize(pGroupResInfo->pRows) > 0) { break; } - assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); + taosArrayClear(pGroupResInfo->pRows); + + pGroupResInfo->index = 0; + pGroupResInfo->rowId = 0; } - SGroupResInfo* info = &pQInfo->groupResInfo; - if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) { + if (pQInfo->groupIndex == numOfGroups && taosArrayGetSize(pGroupResInfo->pRows) == 0) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -3268,89 +3144,28 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { return TSDB_CODE_SUCCESS; } +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t* index, int32_t orderType); + void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - // all results have been return to client, try next group - if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) { - pGroupResInfo->numOfDataPages = 0; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; - + // all results in current group have been returned to client, try next group + if (pGroupResInfo->index >= taosArrayGetSize(pGroupResInfo->pRows)) { // current results of group has been sent to client, try next group - if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { + if (mergeGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { return; // failed to save data in the disk } // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (taosArrayGetSize(pGroupResInfo->pRows) == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } } - SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - int32_t id = pQInfo->groupResInfo.groupId; - SIDList list = getDataBufPagesIdList(pResultBuf, id); - - int32_t offset = 0; - int32_t numOfCopiedRows = 0; - - size_t size = taosArrayGetSize(list); - assert(size == pGroupResInfo->numOfDataPages); - - bool done = false; - - //TODO add API for release none-dirty pages -// SPageInfo* prev = NULL; - - for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { - SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); - - // release previous buffer pages -// if (prev == NULL) { -// prev = pi; -// } else { -// if (prev->pageId != pi->pageId) { -// releaseResBufPageInfo(pResultBuf, prev); -// prev = pi; -// } -// } - - assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); - int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); - - if (numOfRes > pQuery->rec.capacity - offset) { - numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); - pGroupResInfo->rowId += numOfCopiedRows; - done = true; - } else { - numOfCopiedRows = (int32_t)pData->num; - - pGroupResInfo->pageId += 1; - pGroupResInfo->rowId = 0; - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - char * pDest = pQuery->sdata[i]->data; - - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, - (size_t)bytes * numOfCopiedRows); - } - - offset += numOfCopiedRows; - if (done) { - break; - } - } - - assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; + int32_t size = (int32_t) taosArrayGetSize(pGroupResInfo->pRows); + pQuery->rec.rows = doCopyToSData(pQInfo, pGroupResInfo->pRows->pData, (int32_t) size, &pGroupResInfo->index, TSDB_ORDER_ASC); } int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { @@ -3378,155 +3193,99 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResu return 0; } -int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { +int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery); - size_t size = taosArrayGetSize(pGroup); - tFilePage **buffer = pQuery->sdata; + int32_t code = TSDB_CODE_SUCCESS; - int32_t *posList = calloc(size, sizeof(int32_t)); - STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); + int32_t *posList = NULL; + SLoserTreeInfo *pTree = NULL; + STableQueryInfo **pTableQueryInfoList = NULL; + + size_t size = taosArrayGetSize(pTableList); + if (pGroupResInfo->pRows == NULL) { + pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); + } - if (pTableList == NULL || posList == NULL) { - tfree(posList); - tfree(pTableList); + posList = calloc(size, sizeof(int32_t)); + pTableQueryInfoList = malloc(POINTER_BYTES * size); + if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL) { qError("QInfo:%p failed alloc memory", pQInfo); - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; } - // todo opt for the case of one table per group int32_t numOfTables = 0; - SIDList pageList = NULL; - int32_t tid = -1; - for (int32_t i = 0; i < size; ++i) { - STableQueryInfo *item = taosArrayGetP(pGroup, i); - - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); - if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables++] = item; - tid = TSDB_TABLEID(item->pTable)->tid; - pageList = list; + STableQueryInfo *item = taosArrayGetP(pTableList, i); + if (item->windowResInfo.size > 0) { + pTableQueryInfoList[numOfTables++] = item; } } // there is no data in current group + // no need to merge results since only one table in each group if (numOfTables == 0) { - tfree(posList); - tfree(pTableList); - return 0; - } else if (numOfTables == 1) { // no need to merge results since only one table in each group - tfree(posList); - tfree(pTableList); - - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); - pGroupResInfo->groupId = tid; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; - - return pGroupResInfo->numOfDataPages; + goto _end; } - SCompSupporter cs = {pTableList, posList, pQInfo}; + SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQuery->order.order}; - SLoserTreeInfo *pTree = NULL; - tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - - SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - - pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex); + int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); + if (ret != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; + } - // todo add windowRes iterator - int64_t lastTimestamp = -1; + int64_t lastTimestamp = ascQuery? INT64_MIN:INT64_MAX; int64_t startt = taosGetTimestampMs(); while (1) { if (isQueryKilled(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - - tfree(pTableList); - tfree(posList); - tfree(pTree); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + code = TSDB_CODE_TSC_QUERY_CANCELLED; + goto _end; } - int32_t pos = pTree->pNode[0].index; + int32_t tableIndex = pTree->pNode[0].index; - SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); + SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->windowResInfo; + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); - char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); - TSKEY ts = GET_INT64_VAL(b); - - assert(ts == pWindowRes->win.skey); int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); if (num <= 0) { - cs.position[pos] += 1; - - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; + cs.rowIndex[tableIndex] += 1; - // all input sources are exhausted - if (--numOfTables == 0) { + if (cs.rowIndex[tableIndex] >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; + if (--numOfTables == 0) { // all input sources are exhausted break; } } } else { - if (ts == lastTimestamp) { // merge with the last one - doMerge(pRuntimeEnv, ts, pWindowRes, true); - } else { // copy data to disk buffer - if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - return -1; - } + assert((pWindowRes->win.skey >= lastTimestamp && ascQuery) || (pWindowRes->win.skey <= lastTimestamp && !ascQuery)); - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - } - - doMerge(pRuntimeEnv, ts, pWindowRes, false); - buffer[0]->num += 1; + if (pWindowRes->win.skey != lastTimestamp) { + taosArrayPush(pGroupResInfo->pRows, &pWindowRes); + pWindowRes->numOfRows = (uint32_t) num; } - lastTimestamp = ts; + lastTimestamp = pWindowRes->win.skey; - // move to the next element of current entry - int32_t currentPageId = pWindowRes->pageId; - - cs.position[pos] += 1; - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; + // move to the next row of current entry + if ((++cs.rowIndex[tableIndex]) >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; // all input sources are exhausted - if (--numOfTables == 0) { + if ((--numOfTables) == 0) { break; } - } else { - // current page is not needed anymore - SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - if (pNextWindowRes->pageId != currentPageId) { - releaseResBufPage(pRuntimeEnv->pResultBuf, page); - } } } - tLoserTreeAdjust(pTree, pos + pTree->numOfEntries); - } - - if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - - tfree(pTree); - tfree(pTableList); - tfree(posList); - return -1; - } + tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries); } int64_t endt = taosGetTimestampMs(); @@ -3537,65 +3296,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - tfree(pTableList); + _end: + tfree(pTableQueryInfoList); tfree(posList); tfree(pTree); -// tfree(pResultInfo); -// tfree(buf); - - return pQInfo->groupResInfo.numOfDataPages; -} - -int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. - int32_t pageId = -1; - int32_t capacity = pResultBuf->numOfRowsPerPage; - - int32_t remain = (int32_t) pQuery->sdata[0]->num; - int32_t offset = 0; - - while (remain > 0) { - int32_t rows = (remain > capacity)? capacity:remain; - assert(rows > 0); - - // get the output buffer page - tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); - buf->num = rows; - - // pagewisely copy to dest buffer - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - - char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; - char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; - memcpy(output, src, (size_t)(buf->num * bytes)); - } - - offset += rows; - remain -= rows; - - pGroupResInfo->numOfDataPages += 1; - } - - return TSDB_CODE_SUCCESS; -} - -void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; - pCtx[k].size = 1; - pCtx[k].startOffset = 0; - pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k); - - pQuery->sdata[k]->num = 0; - } + return code; } static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { @@ -3603,14 +3309,6 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * return; } - // order has changed already - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { - // do nothing, no results - } else {// NOTE: even win.skey != lastKey, the results may not generated. - pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; - } - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; @@ -3654,7 +3352,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { // group by normal columns and interval query on normal table SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor @@ -3707,8 +3405,8 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { int32_t initResultRow(SResultRow *pResultRow) { pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); - pResultRow->pageId = -1; - pResultRow->rowId = -1; + pResultRow->pageId = -1; + pResultRow->rowId = -1; return TSDB_CODE_SUCCESS; } @@ -3716,7 +3414,7 @@ void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t tid = 0; - int64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -3842,18 +3540,14 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; bool toContinue = false; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pResult = getResultRow(pWindowResInfo, i); - if (!pResult->closed) { - continue; - } setResultOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pExpr1[j].base.functionId; if (functId == TSDB_FUNC_TS) { @@ -3897,10 +3591,6 @@ static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY }; TIME_WINDOW_COPY(info.w, pQuery->window); - TIME_WINDOW_COPY(info.curWindow, pTableQueryInfo->win); - - info.curWindow.skey = start; - return info; } @@ -3908,17 +3598,15 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; - pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor - if (pRuntimeEnv->pTSBuf) { - SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); - bool ret = tsBufNextPos(pRuntimeEnv->pTSBuf); + pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); // save the cursor + if (pRuntimeEnv->pTsBuf) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); assert(ret); } // reverse order time range - pQuery->window = pStatus->curWindow; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -3953,9 +3641,9 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus SWITCH_ORDER(pQuery->order.order); switchCtxOrder(pRuntimeEnv); - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pStatus->cur); - if (pRuntimeEnv->pTSBuf) { - pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pStatus->cur); + if (pRuntimeEnv->pTsBuf) { + pRuntimeEnv->pTsBuf->cur.order = pQuery->order.order; } SET_MASTER_SCAN_FLAG(pRuntimeEnv); @@ -3984,9 +3672,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { // store the start query position SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv, start); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); while (1) { doScanAllDataBlocks(pRuntimeEnv); @@ -3995,13 +3681,9 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan - if (qstatus.lastKey != pTableQueryInfo->lastKey) { - qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; - } else { // the lastkey does not increase, which means no data checked yet + if (qstatus.lastKey == pTableQueryInfo->lastKey) { qDebug("QInfo:%p no results generated in this scan", pQInfo); } - - qstatus.lastKey = pTableQueryInfo->lastKey; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { @@ -4017,7 +3699,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &qstatus.curWindow); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { @@ -4053,16 +3735,16 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol) { - closeAllTimeWindow(pWindowResInfo); + if (pRuntimeEnv->groupbyColumn) { + closeAllResultRows(pWindowResInfo); } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *buf = pWindowResInfo->pResult[i]; - if (!isWindowResClosed(pWindowResInfo, i)) { + if (!isResultRowClosed(pWindowResInfo, i)) { continue; } @@ -4110,9 +3792,9 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void pTableQueryInfo->cur.vgroupIndex = -1; // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { int32_t initialSize = 128; - int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); + int32_t code = initResultRowInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4128,7 +3810,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { } tVariantDestroy(&pTableQueryInfo->tag); - cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); + cleanupResultRowInfo(&pTableQueryInfo->windowResInfo); } /** @@ -4137,14 +3819,14 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { * @param pDataBlockInfo */ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); } @@ -4152,12 +3834,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - uint64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); - if (pResultRow == NULL) { - return; - } + assert (pResultRow != NULL); /* * not assign result buffer yet, add new result buffer @@ -4233,13 +3913,13 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ setTagVal(pRuntimeEnv, pTable, pQInfo->tsdb); // both the master and supplement scan needs to set the correct ts comp start position - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; if (pTableQueryInfo->cur.vgroupIndex == -1) { tVariantAssign(&pTableQueryInfo->tag, pTag); - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, &pTableQueryInfo->tag); // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { @@ -4253,7 +3933,7 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ } // keep the cursor info of current meter - pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { @@ -4261,7 +3941,7 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ } } else { - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); @@ -4303,7 +3983,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { /** * In handling the both ascending and descending order super table query, we need to find the first qualified * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional * operations involve. */ STimeWindow w = TSWINDOW_INITIALIZER; @@ -4312,7 +3992,6 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { TSKEY sk = MIN(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); - pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { if (!QUERY_IS_ASC_QUERY(pQuery)) { @@ -4351,36 +4030,33 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) { +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t *index, int32_t orderType) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; int32_t numOfResult = 0; - int32_t startIdx = 0; + int32_t start = 0; int32_t step = -1; qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); - int32_t totalSet = numOfClosedTimeWindow(pResultInfo); - SResultRow** result = pResultInfo->pResult; - if (orderType == TSDB_ORDER_ASC) { - startIdx = pQInfo->groupIndex; + start = (*index); step = 1; } else { // desc order copy all data - startIdx = totalSet - pQInfo->groupIndex - 1; + start = numOfRows - (*index) - 1; step = -1; } SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { - if (result[i]->numOfRows == 0) { - pQInfo->groupIndex += 1; + for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { + if (pRows[i]->numOfRows == 0) { + (*index) += 1; pGroupResInfo->rowId = 0; continue; } - int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId; + int32_t numOfRowsToCopy = pRows[i]->numOfRows - pGroupResInfo->rowId; int32_t oldOffset = pGroupResInfo->rowId; /* @@ -4392,16 +4068,16 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_ pGroupResInfo->rowId += numOfRowsToCopy; } else { pGroupResInfo->rowId = 0; - pQInfo->groupIndex += 1; + (*index) += 1; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRows[i]->pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page); + char *in = getPosInResultPage(pRuntimeEnv, j, pRows[i], page); memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); } @@ -4432,10 +4108,9 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo, orderType); + int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo->pResult, pResultInfo->size, &pQInfo->groupIndex, orderType); pQuery->rec.rows += numOfResult; - assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -4467,25 +4142,17 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo * pResultRowInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - - // TODO refactor - if ((pTableQueryInfo->lastKey >= pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey <= pTableQueryInfo->win.ekey && (!ascQuery))) { - closeAllTimeWindow(pWindowResInfo); - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - } else { - updateResultRowCurrentIndex(pWindowResInfo, pTableQueryInfo->lastKey, ascQuery); - } + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); } } @@ -4521,7 +4188,7 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { } else { // there are results waiting for returned to client. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && - (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) && + (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) && (pRuntimeEnv->windowResInfo.size > 0)) { return true; } @@ -4557,7 +4224,9 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); + int32_t total = 0; STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL); + while(item) { STableIdInfo* pDst = (STableIdInfo*)data; pDst->uid = htobe64(item->uid); @@ -4565,9 +4234,14 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data pDst->key = htobe64(item->key); data += sizeof(STableIdInfo); + total++; + + qDebug("QInfo:%p set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo, item->tid, item->uid, item->key); item = taosHashIterate(pQInfo->arrTableIdInfo, item); } + qDebug("QInfo:%p set %d subscribe info", pQInfo, total); + // Check if query is completed or not for stable query or normal table query respectively. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (pQInfo->runtimeEnv.stableQuery) { @@ -4766,8 +4440,9 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w return key; } else { // do nothing - pQuery->window.skey = tw.skey; + pQuery->window.skey = tw.skey; pWindowResInfo->prevSKey = tw.skey; + pTableQueryInfo->lastKey = tw.skey; return tw.skey; } @@ -4777,26 +4452,14 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { SQuery *pQuery = pRuntimeEnv->pQuery; - - // get the first unclosed time window - bool assign = false; - for(int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - if (pRuntimeEnv->windowResInfo.pResult[i]->closed) { - continue; - } - - assign = true; - *start = pRuntimeEnv->windowResInfo.pResult[i]->win.skey; - } - - if (!assign) { - *start = pQuery->current->lastKey; + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert(*start <= pQuery->current->lastKey); + } else { + assert(*start >= pQuery->current->lastKey); } - assert(*start <= pQuery->current->lastKey); - // if queried with value filter, do NOT forward query start position - if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { + if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { return true; } @@ -4808,6 +4471,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4819,13 +4483,10 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { if (QUERY_IS_ASC_QUERY(pQuery)) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } } else { getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } @@ -4835,19 +4496,25 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { while (pQuery->limit.offset > 0) { STimeWindow tw = win; - if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; pWindowResInfo->prevSKey = win.skey; + + // current time window is aligned with blockInfo.window.ekey + // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; + if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { + pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + } } - // current window does not ended in current data block, try next data block - getNextTimeWindow(pQuery, &tw); if (pQuery->limit.offset == 0) { *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); return true; } + // current window does not ended in current data block, try next data block + getNextTimeWindow(pQuery, &tw); + /* * If the next time window still starts from current data block, * load the primary timestamp column first, and then find the start position for the next queried time window. @@ -4855,13 +4522,12 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required * time window resides in current data block. */ - if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { + + SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; } @@ -4928,7 +4594,6 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) // update the query time window pQuery->window = cond.twindow; - if (pQInfo->tableGroupInfo.numOfTables == 0) { pQInfo->tableqinfoGroupInfo.numOfTables = 0; } else { @@ -5001,15 +4666,15 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pQInfo->vgId = vgId; pRuntimeEnv->pQuery = pQuery; - pRuntimeEnv->pTSBuf = pTsBuf; + pRuntimeEnv->pTsBuf = pTsBuf; pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->groupbyNormalCol = isGroupbyNormalCol(pQuery->pGroupbyExpr); + pRuntimeEnv->groupbyColumn = isGroupbyNormalCol(pQuery->pGroupbyExpr); if (pTsBuf != NULL) { - int16_t order = (pQuery->order.order == pRuntimeEnv->pTSBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; - tsBufSetTraverseOrder(pRuntimeEnv->pTSBuf, order); + int16_t order = (pQuery->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); } int32_t ps = DEFAULT_PAGE_SIZE; @@ -5025,18 +4690,18 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; + if (pRuntimeEnv->groupbyColumn) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_INT; // group id } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, type); + code = initResultRowInfo(&pRuntimeEnv->windowResInfo, 8, type); if (code != TSDB_CODE_SUCCESS) { return code; } } - } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { + } else if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { int32_t numOfResultRows = getInitialPageNum(pQInfo); getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); @@ -5045,13 +4710,13 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->groupbyColumn) { type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, type); + code = initResultRowInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5103,7 +4768,7 @@ static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTa TSKEY nextKey = pBlockInfo->window.skey; setIntervalQueryRange(pQInfo, nextKey); - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); } } @@ -5151,7 +4816,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { pQuery->current = *pTableQueryInfo; doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); } @@ -5196,7 +4861,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { SArray *group = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(group, index); - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb); } @@ -5234,11 +4899,11 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { longjmp(pRuntimeEnv->env, terrno); } - if (pRuntimeEnv->pTSBuf != NULL) { + if (pRuntimeEnv->pTsBuf != NULL) { tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { @@ -5249,7 +4914,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return false; } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, @@ -5260,10 +4925,10 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } } } else { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); if (tVariantCompare(elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { - STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem1)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { @@ -5274,7 +4939,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return false; } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); } else { @@ -5283,8 +4948,8 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } } else { - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pRuntimeEnv->cur); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); } else { @@ -5409,7 +5074,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } } - } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query + } else if (pRuntimeEnv->groupbyColumn) { // group-by on normal columns query while (pQInfo->groupIndex < numOfGroups) { SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); @@ -5448,10 +5113,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { scanMultiTableDataBlocks(pQInfo); pQInfo->groupIndex += 1; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + taosArrayDestroy(s); // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pWindowResInfo->size <= 0) { continue; } @@ -5478,11 +5143,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQInfo->groupIndex = currentGroupIndex; // restore the group index assert(pQuery->rec.rows == pWindowResInfo->size); - - clearClosedTimeWindow(pRuntimeEnv); + resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); break; } - } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL && !isTSCompQuery(pQuery)) { + } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTsBuf == NULL && !isTSCompQuery(pQuery)) { //super table projection query with identical query time range for all tables. SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; resetDefaultResInfoOutputBuf(pRuntimeEnv); @@ -5641,7 +5305,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } resetDefaultResInfoOutputBuf(pRuntimeEnv); - resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); + resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); SArray *group = GET_TABLEGROUP(pQInfo, 0); assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && @@ -5693,8 +5357,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } - if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + if (pRuntimeEnv->pTsBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; } } else { @@ -5729,8 +5393,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { finalizeQueryResult(pRuntimeEnv); } - if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + if (pRuntimeEnv->pTsBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; } qDebug("QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 @@ -5740,7 +5404,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } -static void doSaveContext(SQInfo *pQInfo) { +static int32_t doSaveContext(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -5748,8 +5412,8 @@ static void doSaveContext(SQInfo *pQInfo) { SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); - if (pRuntimeEnv->pTSBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); + if (pRuntimeEnv->pTsBuf != NULL) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); } STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); @@ -5766,9 +5430,7 @@ static void doSaveContext(SQInfo *pQInfo) { pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + return (pRuntimeEnv->pSecQueryHandle == NULL)? -1:0; } static void doRestoreContext(SQInfo *pQInfo) { @@ -5776,9 +5438,10 @@ static void doRestoreContext(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + SWITCH_ORDER(pQuery->order.order); - if (pRuntimeEnv->pTSBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); + if (pRuntimeEnv->pTsBuf != NULL) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); } switchCtxOrder(pRuntimeEnv); @@ -5796,17 +5459,17 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { size_t num = taosArrayGetSize(group); for (int32_t j = 0; j < num; ++j) { STableQueryInfo* item = taosArrayGetP(group, j); - closeAllTimeWindow(&item->windowResInfo); + closeAllResultRows(&item->windowResInfo); } } } else { // close results for group result - closeAllTimeWindow(&pQInfo->runtimeEnv.windowResInfo); + closeAllResultRows(&pQInfo->runtimeEnv.windowResInfo); } } static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; if (pQInfo->groupIndex > 0) { /* @@ -5815,9 +5478,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ if (QUERY_IS_INTERVAL_QUERY(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif } else { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); } @@ -5843,12 +5503,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { doCloseAllTimeWindowAfterScan(pQInfo); if (needReverseScan(pQuery)) { - doSaveContext(pQInfo); - - el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); - - doRestoreContext(pQInfo); + int32_t code = doSaveContext(pQInfo); + if (code == TSDB_CODE_SUCCESS) { + el = scanMultiTableDataBlocks(pQInfo); + qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); + doRestoreContext(pQInfo); + } else { + pQInfo->code = code; + } } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); } @@ -5862,12 +5524,15 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { - if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { + int32_t code = mergeGroupResult(pQInfo); + if (code == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif + } else { // set the error code + pQInfo->code = code; } } else { // not a interval query copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); @@ -5877,7 +5542,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qDebug("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *) param; SExprInfo* pExprInfo = (SExprInfo*) pSupport->exprList; @@ -5931,7 +5595,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { } } else { arithSup.pArithExpr = pExpr; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, getArithemicInputSrc); } } @@ -5954,7 +5618,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; * select count(*) from table_name group by status_column; */ -static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableAggregationProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -5979,7 +5643,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) limitResults(pRuntimeEnv); } -static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableProjectionProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -6034,33 +5698,6 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) } } -static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - while (1) { - scanOneTableDataBlocks(pRuntimeEnv, start); - - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - finalizeQueryResult(pRuntimeEnv); - - // here we can ignore the records in case of no interpolation - // todo handle offset, in case of top/bottom interval query - if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) && pQuery->limit.offset > 0 && - pQuery->fillType == TSDB_FILL_NONE) { - // maxOutput <= 0, means current query does not generate any results - int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo); - - int32_t c = (int32_t)(MIN(numOfClosed, pQuery->limit.offset)); - clearFirstNWindowRes(pRuntimeEnv, c); - pQuery->limit.offset -= c; - } - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) { - break; - } - } -} - // handle time interval query on table static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); @@ -6068,11 +5705,10 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - int32_t numOfFilled = 0; - TSKEY newStartKey = TSKEY_INITIAL_VAL; + TSKEY newStartKey = QUERY_IS_ASC_QUERY(pQuery)? INT64_MIN:INT64_MAX; // skip blocks without load the actual data block from file if no filter condition present - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { setQueryStatus(pQuery, QUERY_COMPLETED); @@ -6080,59 +5716,40 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { } } - while (1) { - tableIntervalProcessImpl(pRuntimeEnv, newStartKey); + scanOneTableDataBlocks(pRuntimeEnv, newStartKey); + assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - pQInfo->groupIndex = 0; // always start from 0 - pQuery->rec.rows = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + finalizeQueryResult(pRuntimeEnv); - clearFirstNWindowRes(pRuntimeEnv, pQInfo->groupIndex); - } + // skip offset result rows + pQuery->rec.rows = 0; - // no result generated, abort - if (pQuery->rec.rows == 0 || pRuntimeEnv->groupbyNormalCol) { - break; + // not fill or no result generated during this query + if (pQuery->fillType == TSDB_FILL_NONE || pRuntimeEnv->windowResInfo.size == 0) { + // all data scanned, the group by normal column can return + int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); + if (pQuery->limit.offset > numOfClosed) { + return; } - doSecondaryArithmeticProcess(pQuery); - - // the offset is handled at prepare stage if no interpolation involved - if (pQuery->fillType == TSDB_FILL_NONE) { - limitResults(pRuntimeEnv); - break; - } else { - taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); - taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata); - numOfFilled = 0; + pQInfo->groupIndex = (int32_t) pQuery->limit.offset; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); - if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - limitResults(pRuntimeEnv); - break; - } + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + doSecondaryArithmeticProcess(pQuery); - // no result generated yet, continue retrieve data - pQuery->rec.rows = 0; - } - } + limitResults(pRuntimeEnv); + } else { - // all data scanned, the group by normal column can return - if (pRuntimeEnv->groupbyNormalCol) { // todo refactor with merge interval time result - // maxOutput <= 0, means current query does not generate any results - int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + doSecondaryArithmeticProcess(pQuery); - if ((pQuery->limit.offset > 0 && pQuery->limit.offset < numOfClosed) || pQuery->limit.offset == 0) { - // skip offset result rows - clearFirstNWindowRes(pRuntimeEnv, (int32_t) pQuery->limit.offset); + taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); + taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage **)pQuery->sdata); - pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - clearFirstNWindowRes(pRuntimeEnv, pQInfo->groupIndex); + int32_t numOfFilled = 0; + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); - doSecondaryArithmeticProcess(pQuery); + if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); } } @@ -6143,7 +5760,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; if (queryHasRemainResForTableQuery(pRuntimeEnv)) { - if (pQuery->fillType != TSDB_FILL_NONE) { /* * There are remain results that are not returned due to result interpolation @@ -6160,23 +5776,23 @@ static void tableQueryImpl(SQInfo *pQInfo) { return; } else { pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; // always start from 0 + assert(pRuntimeEnv->windowResInfo.size > 0); - if (pRuntimeEnv->windowResInfo.size > 0) { + if (pQInfo->groupIndex < pRuntimeEnv->windowResInfo.size) { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - clearFirstNWindowRes(pRuntimeEnv, pQInfo->groupIndex); - - if (pQuery->rec.rows > 0) { - qDebug("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); + } - // there are not data remains - if (pRuntimeEnv->windowResInfo.size <= 0) { - qDebug("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total); - } + if (pQuery->rec.rows > 0) { + qDebug("QInfo:%p %" PRId64 " rows returned from group results, total:%" PRId64 "", pQInfo, pQuery->rec.rows, + pQuery->rec.total); + } - return; - } + // there are not data remains + if (pQuery->rec.rows <= 0 || pRuntimeEnv->windowResInfo.size <= pQInfo->groupIndex) { + qDebug("QInfo:%p query over, %" PRId64 " rows are returned", pQInfo, pQuery->rec.total); } + + return; } } @@ -6189,13 +5805,13 @@ static void tableQueryImpl(SQInfo *pQInfo) { STableQueryInfo* item = taosArrayGetP(g, 0); // group by normal column, sliding window query, interval query are handled by interval query processor - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); } else if (isFixedOutputQuery(pRuntimeEnv)) { - tableFixedOutputProcess(pQInfo, item); + tableAggregationProcess(pQInfo, item); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); - tableMultiOutputProcess(pQInfo, item); + tableProjectionProcess(pQInfo, item); } // record the total elapsed time @@ -6211,11 +5827,11 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyColumn))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || - isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); + pRuntimeEnv->groupbyColumn); sequentialTableProcess(pQInfo); } @@ -6251,13 +5867,13 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } } - assert(0); - return -1; + + return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value } bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg, pTagCols); - return j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags; + return j != INT32_MIN; } static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { @@ -6284,12 +5900,14 @@ static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { return true; } -static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg) { +static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg, SColumnInfo* pTagCols) { int32_t numOfTotal = pQueryMsg->numOfCols + pQueryMsg->numOfTags; if (pQueryMsg->numOfCols < 0 || pQueryMsg->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pQueryMsg, pQueryMsg->numOfCols, pQueryMsg->numOfTags); return false; - } else if (numOfTotal == 0) { + } + + if (numOfTotal == 0) { for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { SSqlFuncMsg* pFuncMsg = pExprMsg[i]; @@ -6303,6 +5921,12 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx } } + for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + if (!validateExprColumnInfo(pQueryMsg, pExprMsg[i], pTagCols)) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } + return true; } @@ -6334,7 +5958,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, SSqlFuncMsg ***pSecStageExpr, - char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { + char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols, char** sql) { int32_t code = TSDB_CODE_SUCCESS; if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -6366,7 +5990,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); + pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); + pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); // query msg safety check if (!validateQueryMsg(pQueryMsg)) { @@ -6461,20 +6087,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; } - if (!validateQuerySourceCols(pQueryMsg, *pExpr)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - if (pQueryMsg->secondStageOutput) { pExprMsg = (SSqlFuncMsg *)pMsg; *pSecStageExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); @@ -6508,10 +6125,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; @@ -6528,13 +6141,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - (*groupbyCols)[i].colId = *(int16_t *)pMsg; + (*groupbyCols)[i].colId = htons(*(int16_t *)pMsg); pMsg += sizeof((*groupbyCols)[i].colId); - (*groupbyCols)[i].colIndex = *(int16_t *)pMsg; + (*groupbyCols)[i].colIndex = htons(*(int16_t *)pMsg); pMsg += sizeof((*groupbyCols)[i].colIndex); - (*groupbyCols)[i].flag = *(int16_t *)pMsg; + (*groupbyCols)[i].flag = htons(*(int16_t *)pMsg); pMsg += sizeof((*groupbyCols)[i].flag); memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name)); @@ -6590,17 +6203,22 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += pQueryMsg->tagCondLen; } - if (*pMsg != 0) { - size_t len = strlen(pMsg) + 1; - - *tbnameCond = malloc(len); + if (pQueryMsg->tbnameCondLen > 0) { + *tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); if (*tbnameCond == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } - strcpy(*tbnameCond, pMsg); - pMsg += len; + strncpy(*tbnameCond, pMsg, pQueryMsg->tbnameCondLen); + pMsg += pQueryMsg->tbnameCondLen; + } + + *sql = strndup(pMsg, pQueryMsg->sqlstrLen); + + if (!validateQuerySourceCols(pQueryMsg, *pExpr, *tagCols)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; } qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " @@ -6609,6 +6227,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); + qDebug("qmsg:%p, sql:%s", pQueryMsg, *sql); return TSDB_CODE_SUCCESS; _cleanup: @@ -6619,6 +6238,7 @@ _cleanup: tfree(*groupbyCols); tfree(*tagCols); tfree(*tagCond); + tfree(*sql); return code; } @@ -6691,7 +6311,15 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num } } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); + if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { + if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pQueryMsg->numOfTags) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } else { + if (j < PRIMARYKEY_TIMESTAMP_COL_INDEX || j >= pQueryMsg->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; @@ -6715,6 +6343,7 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExprs[i].bytes; } + assert(isValidDataType(pExprs[i].type)); } @@ -6917,7 +6546,7 @@ static void calResultBufSize(SQuery* pQuery) { } static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery) { + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -7029,7 +6658,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); - pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv)); + pQInfo->runtimeEnv.pool = initResultRowPool(getResultRowSize(&pQInfo->runtimeEnv)); pQInfo->runtimeEnv.prevRow = malloc(POINTER_BYTES * pQuery->numOfCols + srcSize); char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pQInfo->runtimeEnv.prevRow; @@ -7048,6 +6677,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; + pQInfo->sql = sql; pthread_mutex_init(&pQInfo->lock, NULL); tsem_init(&pQInfo->ready, 0, 0); @@ -7291,9 +6921,13 @@ static void freeQInfo(SQInfo *pQInfo) { doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); tfree(pQInfo->pBuf); + tfree(pQInfo->sql); + tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosHashCleanup(pQInfo->arrTableIdInfo); + taosArrayDestroy(pQInfo->groupResInfo.pRows); + pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -7387,6 +7021,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi int32_t code = TSDB_CODE_SUCCESS; + char *sql = NULL; char *tagCond = NULL; char *tbnameCond = NULL; SArray *pTableIdList = NULL; @@ -7399,7 +7034,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi SColumnInfo *pTagColumnInfo = NULL; SSqlGroupbyExpr *pGroupbyExpr = NULL; - code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo); + code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo, &sql); if (code != TSDB_CODE_SUCCESS) { goto _over; } @@ -7483,8 +7118,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi goto _over; } - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery); + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery, sql); + sql = NULL; pExprs = NULL; pSecExprs = NULL; pGroupbyExpr = NULL; @@ -7508,6 +7144,7 @@ _over: } free(pTagColumnInfo); + free(sql); free(pExprs); free(pSecExprs); @@ -7551,7 +7188,7 @@ static bool doBuildResCheck(SQInfo* pQInfo) { // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is // put into task to be executed. - assert(pQInfo->owner == taosGetPthreadId()); + assert(pQInfo->owner == taosGetSelfPthreadId()); pQInfo->owner = 0; pthread_mutex_unlock(&pQInfo->lock); @@ -7564,7 +7201,7 @@ static bool doBuildResCheck(SQInfo* pQInfo) { bool qTableQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; assert(pQInfo && pQInfo->signature == pQInfo); - int64_t threadId = taosGetPthreadId(); + int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { @@ -7635,7 +7272,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex int32_t code = TSDB_CODE_SUCCESS; - if (tsHalfCoresForQuery) { + if (tsRetrieveBlockingModel) { pQInfo->rspContext = pRspContext; tsem_wait(&pQInfo->ready); *buildRes = true; @@ -7713,7 +7350,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->completed = 1; // notify no more result to client } else { *continueExec = true; - qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); + qDebug("QInfo:%p has more results to retrieve", pQInfo); } return pQInfo->code; @@ -7991,9 +7628,9 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) { SQueryMgmt* pQueryMgmt = pQMgmt; qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); -// pthread_mutex_lock(&pQueryMgmt->lock); + pthread_mutex_lock(&pQueryMgmt->lock); pQueryMgmt->closed = true; -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); } @@ -8031,9 +7668,9 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { return NULL; } -// pthread_mutex_lock(&pQueryMgmt->lock); + pthread_mutex_lock(&pQueryMgmt->lock); if (pQueryMgmt->closed) { -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; return NULL; @@ -8041,7 +7678,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo; void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), (getMaximumIdleDurationSec()*1000)); -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); return handle; } @@ -8078,6 +7715,4 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); return 0; -} - - +} \ No newline at end of file diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index b6050dddd8fecc8f0a0b764313efcbe2a630113a..2a40533e90218170a465a4f439477e102123157f 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -21,6 +21,12 @@ #include "tcompare.h" #include "tsqlfunction.h" +#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (4 * FLT_EPSILON)) +#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y))) +#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y))) +#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) +#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) + bool less_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { return (*(int8_t *)minval < pFilter->filterInfo.upperBndi); } @@ -38,35 +44,35 @@ bool less_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { } bool less_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval < pFilter->filterInfo.upperBndd); + return FLT_LESS(*(float*)minval, pFilter->filterInfo.upperBndd); } bool less_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)minval < pFilter->filterInfo.upperBndd); + return *(double *)minval < pFilter->filterInfo.upperBndd; } ////////////////////////////////////////////////////////////////// -bool large_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { +bool larger_i8(SColumnFilterElem *pFilter, char *minval, char *maxval) { return (*(int8_t *)maxval > pFilter->filterInfo.lowerBndi); } -bool large_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { +bool larger_i16(SColumnFilterElem *pFilter, char *minval, char *maxval) { return (*(int16_t *)maxval > pFilter->filterInfo.lowerBndi); } -bool large_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { +bool larger_i32(SColumnFilterElem *pFilter, char *minval, char *maxval) { return (*(int32_t *)maxval > pFilter->filterInfo.lowerBndi); } -bool large_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { +bool larger_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { return (*(int64_t *)maxval > pFilter->filterInfo.lowerBndi); } -bool large_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)maxval > pFilter->filterInfo.lowerBndd); +bool larger_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { + return FLT_GREATER(*(float*)maxval, pFilter->filterInfo.lowerBndd); } -bool large_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { +bool larger_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { return (*(double *)maxval > pFilter->filterInfo.lowerBndd); } ///////////////////////////////////////////////////////////////////// @@ -88,10 +94,14 @@ bool lessEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { } bool lessEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->filterInfo.upperBndd); + return FLT_LESSEQUAL(*(float*)minval, pFilter->filterInfo.upperBndd); } bool lessEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { + if ((fabs(*(double*)minval) - pFilter->filterInfo.upperBndd) <= 2 * DBL_EPSILON) { + return true; + } + return (*(double *)minval <= pFilter->filterInfo.upperBndd); } @@ -113,11 +123,15 @@ bool largeEqual_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { } bool largeEqual_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)maxval >= pFilter->filterInfo.lowerBndd); + return FLT_GREATEREQUAL(*(float*)maxval, pFilter->filterInfo.lowerBndd); } bool largeEqual_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(double *)maxval >= pFilter->filterInfo.lowerBndd); + if (fabs(*(double *)maxval - pFilter->filterInfo.lowerBndd) <= 2 * DBL_EPSILON) { + return true; + } + + return (*(double *)maxval - pFilter->filterInfo.lowerBndd > (2 * DBL_EPSILON)); } //////////////////////////////////////////////////////////////////////// @@ -162,10 +176,12 @@ bool equal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { } } +// user specified input filter value and the original saved float value may needs to +// increase the tolerance to obtain the correct result. bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { if (*(float *)minval == *(float *)maxval) { - return (fabs(*(float *)minval - pFilter->filterInfo.lowerBndd) <= FLT_EPSILON); - } else { /* range filter */ + return FLT_EQUAL(*(float*)minval, pFilter->filterInfo.lowerBndd); + } else { // range filter assert(*(float *)minval < *(float *)maxval); return *(float *)minval <= pFilter->filterInfo.lowerBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd; } @@ -173,10 +189,9 @@ bool equal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { if (*(double *)minval == *(double *)maxval) { - return (*(double *)minval == pFilter->filterInfo.lowerBndd); - } else { /* range filter */ + return (fabs(*(double *)minval - pFilter->filterInfo.lowerBndd) <= 2 * DBL_EPSILON); + } else { // range filter assert(*(double *)minval < *(double *)maxval); - return *(double *)minval <= pFilter->filterInfo.lowerBndi && *(double *)maxval >= pFilter->filterInfo.lowerBndi; } } @@ -255,7 +270,7 @@ bool nequal_i64(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool nequal_ds(SColumnFilterElem *pFilter, char *minval, char *maxval) { if (*(float *)minval == *(float *)maxval) { - return (*(float *)minval != pFilter->filterInfo.lowerBndd); + return !FLT_EQUAL(*(float *)minval, pFilter->filterInfo.lowerBndd); } return true; @@ -364,7 +379,8 @@ bool rangeFilter_i64_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) //////////////////////////////////////////////////////////////////////// bool rangeFilter_ds_ii(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval >= pFilter->filterInfo.lowerBndd); + return FLT_LESSEQUAL(*(float *)minval, pFilter->filterInfo.upperBndd) && + FLT_GREATEREQUAL(*(float *)maxval, pFilter->filterInfo.lowerBndd); } bool rangeFilter_ds_ee(SColumnFilterElem *pFilter, char *minval, char *maxval) { @@ -376,7 +392,8 @@ bool rangeFilter_ds_ie(SColumnFilterElem *pFilter, char *minval, char *maxval) { } bool rangeFilter_ds_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { - return (*(float *)minval <= pFilter->filterInfo.upperBndd && *(float *)maxval > pFilter->filterInfo.lowerBndd); + return FLT_GREATER(*(float *)maxval, pFilter->filterInfo.lowerBndd) && + FLT_LESSEQUAL(*(float *)minval, pFilter->filterInfo.upperBndd); } ////////////////////////////////////////////////////////////////////////// @@ -400,7 +417,7 @@ bool rangeFilter_dd_ei(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { NULL, less_i8, - large_i8, + larger_i8, equal_i8, lessEqual_i8, largeEqual_i8, @@ -413,7 +430,7 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { NULL, less_i16, - large_i16, + larger_i16, equal_i16, lessEqual_i16, largeEqual_i16, @@ -426,7 +443,7 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { NULL, less_i32, - large_i32, + larger_i32, equal_i32, lessEqual_i32, largeEqual_i32, @@ -439,7 +456,7 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { NULL, less_i64, - large_i64, + larger_i64, equal_i64, lessEqual_i64, largeEqual_i64, @@ -452,7 +469,7 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { NULL, less_ds, - large_ds, + larger_ds, equal_ds, lessEqual_ds, largeEqual_ds, @@ -465,7 +482,7 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = { NULL, less_dd, - large_dd, + larger_dd, equal_dd, lessEqual_dd, largeEqual_dd, @@ -551,7 +568,7 @@ bool (*rangeFilterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *max __filter_func_t* getRangeFilterFuncArray(int32_t type) { switch(type) { - case TSDB_DATA_TYPE_BOOL: return rangeFilterFunc_i8; + case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: return rangeFilterFunc_i8; case TSDB_DATA_TYPE_SMALLINT: return rangeFilterFunc_i16; case TSDB_DATA_TYPE_INT: return rangeFilterFunc_i32; @@ -565,7 +582,7 @@ __filter_func_t* getRangeFilterFuncArray(int32_t type) { __filter_func_t* getValueFilterFuncArray(int32_t type) { switch(type) { - case TSDB_DATA_TYPE_BOOL: return filterFunc_i8; + case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: return filterFunc_i8; case TSDB_DATA_TYPE_SMALLINT: return filterFunc_i16; case TSDB_DATA_TYPE_INT: return filterFunc_i32; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 35e5906d1f797c2f53f3906bf93c6797ac766c3e..0d31948e59e520452605ac4577b695c53b3fad6c 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -158,8 +158,8 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } #if defined(USE_ARRAYLIST) - int32_t idx = vnodeHistobinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries); + int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); + assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); if ((*pHisto)->elems[idx].val == val && idx >= 0) { (*pHisto)->elems[idx].num += 1; @@ -174,7 +174,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } assert((*pHisto)->elems[idx].val > val); - } else { + } else if ((*pHisto)->numOfElems > 0) { assert((*pHisto)->elems[(*pHisto)->numOfEntries].val < val); } @@ -356,7 +356,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { return 0; } -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val) { +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { int32_t end = len - 1; int32_t start = 0; @@ -466,7 +466,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) { */ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { #if defined(USE_ARRAYLIST) - int32_t slotIdx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, v); + int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v); if (pHisto->elems[slotIdx].val != v) { slotIdx -= 1; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 7d71d9f7f15ad88d0bc1565047ae7686320d2832..4b3a56e4ee503bfcd8a558bfe9050a787310650e 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -71,13 +71,13 @@ abort_parse: return sqlInfo; } -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { +tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { if (pList == NULL) { pList = calloc(1, sizeof(tSQLExprList)); } if (pList->nAlloc <= pList->nExpr) { - pList->nAlloc = (pList->nAlloc << 1) + 4; + pList->nAlloc = (pList->nAlloc << 1u) + 4; pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); if (pList->a == 0) { pList->nExpr = pList->nAlloc = 0; @@ -87,7 +87,7 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken assert(pList->a != 0); if (pNode || pToken) { - struct tSQLExprItem *pItem = &pList->a[pList->nExpr++]; + struct tSqlExprItem *pItem = &pList->a[pList->nExpr++]; memset(pItem, 0, sizeof(*pItem)); pItem->pNode = pNode; if (pToken) { // set the as clause @@ -101,62 +101,62 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken return pList; } -void tSQLExprListDestroy(tSQLExprList *pList) { +void tSqlExprListDestroy(tSQLExprList *pList) { if (pList == NULL) return; for (int32_t i = 0; i < pList->nExpr; ++i) { if (pList->a[i].aliasName != NULL) { free(pList->a[i].aliasName); } - tSQLExprDestroy(pList->a[i].pNode); + tSqlExprDestroy(pList->a[i].pNode); } free(pList->a); free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { - tSQLExpr *pSQLExpr = calloc(1, sizeof(tSQLExpr)); +tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType) { + tSQLExpr *pSqlExpr = calloc(1, sizeof(tSQLExpr)); if (pToken != NULL) { - pSQLExpr->token = *pToken; + pSqlExpr->token = *pToken; } if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { toTSDBType(pToken->type); - tVariantCreate(&pSQLExpr->val, pToken); - pSQLExpr->nSQLOptr = optrType; + tVariantCreate(&pSqlExpr->val, pToken); + pSqlExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // use microsecond by default - pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSqlExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSQLExpr->val.i64Key); + int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->val.i64Key); if (ret != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSQLExpr->nSQLOptr = TK_TIMESTAMP; + pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->nSQLOptr = TK_TIMESTAMP; } else { // it must be the column name (tk_id) if it is not the number assert(optrType == TK_ID || optrType == TK_ALL); if (pToken != NULL) { - pSQLExpr->colInfo = *pToken; + pSqlExpr->colInfo = *pToken; } - pSQLExpr->nSQLOptr = optrType; + pSqlExpr->nSQLOptr = optrType; } - return pSQLExpr; + return pSqlExpr; } /* * pList is the parameters for function with id(optType) * function name is denoted by pFunctionToken */ -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { +tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { if (pFuncToken == NULL) return NULL; tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); @@ -177,7 +177,7 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt * create binary expression in this procedure * if the expr is arithmetic, calculate the result and set it to tSQLExpr Object */ -tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { +tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { @@ -223,8 +223,8 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { } } - tSQLExprDestroy(pLeft); - tSQLExprDestroy(pRight); + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); } else if ((pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_INTEGER) || (pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_FLOAT) || (pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_FLOAT)) { @@ -257,8 +257,8 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { } } - tSQLExprDestroy(pLeft); - tSQLExprDestroy(pRight); + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); } else { pExpr->nSQLOptr = optrType; @@ -288,7 +288,7 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { return pExpr; } -void tSQLExprNodeDestroy(tSQLExpr *pExpr) { +void tSqlExprNodeDestroy(tSQLExpr *pExpr) { if (pExpr == NULL) { return; } @@ -297,20 +297,20 @@ void tSQLExprNodeDestroy(tSQLExpr *pExpr) { tVariantDestroy(&pExpr->val); } - tSQLExprListDestroy(pExpr->pParam); + tSqlExprListDestroy(pExpr->pParam); free(pExpr); } -void tSQLExprDestroy(tSQLExpr *pExpr) { +void tSqlExprDestroy(tSQLExpr *pExpr) { if (pExpr == NULL) { return; } - tSQLExprDestroy(pExpr->pLeft); - tSQLExprDestroy(pExpr->pRight); + tSqlExprDestroy(pExpr->pLeft); + tSqlExprDestroy(pExpr->pRight); - tSQLExprNodeDestroy(pExpr); + tSqlExprNodeDestroy(pExpr); } SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) { @@ -366,13 +366,13 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int return pList; } -void setDBName(SStrToken *pCpxName, SStrToken *pDB) { - pCpxName->type = pDB->type; - pCpxName->z = pDB->z; - pCpxName->n = pDB->n; +void setDbName(SStrToken *pCpxName, SStrToken *pDb) { + pCpxName->type = pDb->type; + pCpxName->z = pDb->z; + pCpxName->n = pDb->n; } -void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { +void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); // truncate the column name @@ -384,13 +384,18 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) pField->name[pName->n] = 0; pField->type = pType->type; - pField->bytes = pType->bytes; + if(pField->type < TSDB_DATA_TYPE_BOOL || pField->type > TSDB_DATA_TYPE_NCHAR){ + pField->bytes = 0; + } else { + pField->bytes = pType->bytes; + } + } -void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { +void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = -1; - for (int8_t i = 0; i < tListLen(tDataTypeDesc); ++i) { + for (int32_t i = 0; i < tListLen(tDataTypeDesc); ++i) { if ((strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0) && (type->n == tDataTypeDesc[i].nameLen)) { pField->type = i; @@ -438,7 +443,7 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, +SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); @@ -474,21 +479,28 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, return pQuery; } -void freeVariant(void *pItem) { +static void freeVariant(void *pItem) { tVariantListItem* p = (tVariantListItem*) pItem; tVariantDestroy(&p->pVar); } +void freeCreateTableInfo(void* p) { + SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p; + taosArrayDestroyEx(pInfo->pTagVals, freeVariant); + tfree(pInfo->fullname); + tfree(pInfo->tagdata.data); +} + void doDestroyQuerySql(SQuerySQL *pQuerySql) { if (pQuerySql == NULL) { return; } - - tSQLExprListDestroy(pQuerySql->pSelection); + + tSqlExprListDestroy(pQuerySql->pSelection); pQuerySql->pSelection = NULL; - - tSQLExprDestroy(pQuerySql->pWhere); + + tSqlExprDestroy(pQuerySql->pWhere); pQuerySql->pWhere = NULL; taosArrayDestroyEx(pQuerySql->pSortOrder, freeVariant); @@ -519,31 +531,30 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { tfree(pClause->pClause); } -SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pStableName, - SArray *pTagVals, SQuerySQL *pSelect, int32_t type) { +SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type) { SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); switch (type) { case TSQL_CREATE_TABLE: { pCreate->colInfo.pColumns = pCols; - assert(pTagVals == NULL && pTags == NULL); + assert(pTags == NULL); break; } case TSQL_CREATE_STABLE: { pCreate->colInfo.pColumns = pCols; pCreate->colInfo.pTagColumns = pTags; - assert(pTagVals == NULL && pTags != NULL && pCols != NULL); - break; - } - case TSQL_CREATE_TABLE_FROM_STABLE: { - pCreate->usingInfo.pTagVals = pTagVals; - pCreate->usingInfo.stableName = *pStableName; + assert(pTags != NULL && pCols != NULL); break; } case TSQL_CREATE_STREAM: { pCreate->pSelect = pSelect; break; } + + case TSQL_CREATE_TABLE_FROM_STABLE: { + assert(0); + } + default: assert(false); } @@ -552,10 +563,22 @@ SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pSt return pCreate; } -SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type) { +SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists) { + SCreatedTableInfo info; + memset(&info, 0, sizeof(SCreatedTableInfo)); + + info.name = *pToken; + info.pTagVals = pTagVals; + info.stableName = *pTableName; + info.igExist = (igExists->n > 0)? 1:0; + + return info; +} + +SAlterTableSQL *tAlterTableSqlElems(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type) { SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); - pAlterTable->name = *pMeterName; + pAlterTable->name = *pTableName; pAlterTable->type = type; if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { @@ -573,24 +596,29 @@ SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray return pAlterTable; } -void SQLInfoDestroy(SSqlInfo *pInfo) { +void* destroyCreateTableSql(SCreateTableSQL* pCreate) { + doDestroyQuerySql(pCreate->pSelect); + + taosArrayDestroy(pCreate->colInfo.pColumns); + taosArrayDestroy(pCreate->colInfo.pTagColumns); + + taosArrayDestroyEx(pCreate->childTableInfo, freeCreateTableInfo); + tfree(pCreate); + + return NULL; +} + +void SqlInfoDestroy(SSqlInfo *pInfo) { if (pInfo == NULL) return; if (pInfo->type == TSDB_SQL_SELECT) { destroyAllSelectClause(&pInfo->subclauseInfo); } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) { - SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo; - doDestroyQuerySql(pCreateTableInfo->pSelect); - - taosArrayDestroy(pCreateTableInfo->colInfo.pColumns); - taosArrayDestroy(pCreateTableInfo->colInfo.pTagColumns); - - taosArrayDestroyEx(pCreateTableInfo->usingInfo.pTagVals, freeVariant); - tfree(pInfo->pCreateTableInfo); + pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant); taosArrayDestroy(pInfo->pAlterInfo->pAddColumns); - + tfree(pInfo->pAlterInfo->tagData.data); tfree(pInfo->pAlterInfo); } else { if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) { @@ -624,7 +652,7 @@ SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { return pSubclause; } -SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type) { +SSqlInfo*setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type) { pInfo->type = type; if (type == TSDB_SQL_SELECT) { @@ -634,8 +662,8 @@ SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, pInfo->pCreateTableInfo = pSqlExprInfo; } - if (pMeterName != NULL) { - pInfo->pCreateTableInfo->name = *pMeterName; + if (pTableName != NULL) { + pInfo->pCreateTableInfo->name = *pTableName; } return pInfo; @@ -653,14 +681,14 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists) { - pInfo->pCreateTableInfo->name = *pMeterName; +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists) { + pInfo->pCreateTableInfo->name = *pTableNameToken; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { if (pTokenList->nAlloc <= pTokenList->nTokens) { // - pTokenList->nAlloc = (pTokenList->nAlloc << 1) + 4; + pTokenList->nAlloc = (pTokenList->nAlloc << 1u) + 4; pTokenList->a = realloc(pTokenList->a, pTokenList->nAlloc * sizeof(pTokenList->a[0])); if (pTokenList->a == 0) { pTokenList->nTokens = pTokenList->nAlloc = 0; @@ -695,7 +723,7 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { va_end(va); } -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { pInfo->type = type; pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); @@ -735,7 +763,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -751,7 +779,7 @@ void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken } } -void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { +void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->type = TSDB_SQL_CREATE_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -763,7 +791,7 @@ void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->pDCLInfo->user.passwd = *pPasswd; } -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { pInfo->type = TSDB_SQL_ALTER_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -788,7 +816,7 @@ void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* } } -void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { +void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -818,5 +846,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->keep = NULL; pDBInfo->update = -1; + pDBInfo->cachelast = 0; memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index bc7243830d836287cde808076cfb24de80fa574d..d45e76c2fd07f37afa0ffad0f9db65d5e1948c43 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -313,7 +313,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 // allocate buf if (availablePage == NULL) { - pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES); + pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES + 2); // add extract bytes in case of zipped buffer increased. } else { pi->pData = availablePage; } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 98545c8ef3930587b6f7bd2c5b8d40ded0066d52..e243637333af5df05b02ee6e18c3a1387973a21f 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -238,6 +238,7 @@ static SKeyword keywordTable[] = { {"SUM_IRATE", TK_SUM_IRATE}, {"AVG_RATE", TK_AVG_RATE}, {"AVG_IRATE", TK_AVG_IRATE}, + {"CACHELAST", TK_CACHELAST}, }; static const char isIdChar[] = { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 6c845b012faade8fbb693d3406013b8264b0c4a7..dc01de0f92a0d591c67f3a156caaae34e318c962 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -20,18 +20,6 @@ #include "qExecutor.h" #include "qUtil.h" -static int32_t getResultRowKeyInfo(SResultRow* pResult, int16_t type, char** key, int16_t* bytes) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - *key = varDataVal(pResult->key); - *bytes = varDataLen(pResult->key); - } else { - *key = (char*) &pResult->win.skey; - *bytes = tDataTypeDesc[type].nSize; - } - - return 0; -} - int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; @@ -43,7 +31,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { +int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { pResultRowInfo->capacity = size; pResultRowInfo->type = type; @@ -59,10 +47,11 @@ int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t return TSDB_CODE_SUCCESS; } -void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) { +void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { if (pResultRowInfo == NULL) { return; } + if (pResultRowInfo->capacity == 0) { assert(pResultRowInfo->pResult == NULL); return; @@ -77,7 +66,7 @@ void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) { tfree(pResultRowInfo->pResult); } -void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { +void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) { return; } @@ -95,81 +84,10 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultR pResultRowInfo->curIndex = -1; pResultRowInfo->size = 0; - - pResultRowInfo->startTime = TSKEY_INITIAL_VAL; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } -void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { - SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; - if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) { - return; - } - - int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); - assert(num >= 0 && num <= numOfClosed); - - int16_t type = pResultRowInfo->type; - int64_t uid = getResultInfoUId(pRuntimeEnv); - - char *key = NULL; - int16_t bytes = -1; - - for (int32_t i = 0; i < num; ++i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { // remove the window slot from hash table - getResultRowKeyInfo(pResult, type, &key, &bytes); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - } else { - break; - } - } - - int32_t remain = pResultRowInfo->size - num; - - // clear all the closed windows from the window list - for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type); - } - - // move the unclosed window in the front of the window list - for (int32_t k = remain; k < pResultRowInfo->size; ++k) { - SResultRow *pWindowRes = pResultRowInfo->pResult[k]; - clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); - } - - pResultRowInfo->size = remain; - - for (int32_t k = 0; k < pResultRowInfo->size; ++k) { - SResultRow *pResult = pResultRowInfo->pResult[k]; - getResultRowKeyInfo(pResult, type, &key, &bytes); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - - int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - assert(p != NULL); - - int32_t v = (*p - num); - assert(v >= 0 && v <= pResultRowInfo->size); - - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); - } - - pResultRowInfo->curIndex = -1; -} - -void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { - SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; - if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) { - return; - } - - int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); - clearFirstNWindowRes(pRuntimeEnv, numOfClosed); -} - -int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) { +int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { int32_t i = 0; while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { ++i; @@ -178,73 +96,40 @@ int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) { return i; } -void closeAllTimeWindow(SResultRowInfo *pResultRowInfo) { +void closeAllResultRows(SResultRowInfo *pResultRowInfo) { assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - if (pResultRowInfo->pResult[i]->closed) { + SResultRow* pRow = pResultRowInfo->pResult[i]; + if (pRow->closed) { continue; } - pResultRowInfo->pResult[i]->closed = true; + pRow->closed = true; } } -/* - * remove the results that are not the FIRST time window that spreads beyond the - * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time. - * NOTE: remove redundant, only when the result set order equals to traverse order - */ -void removeRedundantWindow(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) { - assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); - if (pResultRowInfo->size <= 1) { - return; - } - - // get the result order - int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1; - if (order != resultOrder) { - return; - } - - int32_t i = 0; - if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey; - while (i < pResultRowInfo->size && (ekey < lastKey)) { - ++i; - } - } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) { - ++i; - } - } - - if (i < pResultRowInfo->size) { - pResultRowInfo->size = (i + 1); - } -} - -bool isWindowResClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { +bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { return (getResultRow(pResultRowInfo, slot)->closed == true); } -void closeTimeWindow(SResultRowInfo *pResultRowInfo, int32_t slot) { +void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { getResultRow(pResultRowInfo, slot)->closed = true; } -void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) { - if (pWindowRes == NULL) { +void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { + if (pResultRow == NULL) { return; } // the result does not put into the SDiskbasedResultBuf, ignore it. - if (pWindowRes->pageId >= 0) { - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); + if (pResultRow->pageId >= 0) { + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { - SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i]; + SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i]; - char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); + char * s = getPosInResultPage(pRuntimeEnv, i, pResultRow, page); size_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes; memset(s, 0, size); @@ -252,56 +137,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16 } } - pWindowRes->numOfRows = 0; - pWindowRes->pageId = -1; - pWindowRes->rowId = -1; - pWindowRes->closed = false; - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tfree(pWindowRes->key); - } else { - pWindowRes->win = TSWINDOW_INITIALIZER; - } -} - -/** - * The source window result pos attribution of the source window result does not assign to the destination, - * since the attribute of "Pos" is bound to each window result when the window result is created in the - * disk-based result buffer. - */ -void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src, int16_t type) { - dst->numOfRows = src->numOfRows; + pResultRow->numOfRows = 0; + pResultRow->pageId = -1; + pResultRow->rowId = -1; + pResultRow->closed = false; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - dst->key = realloc(dst->key, varDataTLen(src->key)); - varDataCopy(dst->key, src->key); + tfree(pResultRow->key); } else { - dst->win = src->win; - } - dst->closed = src->closed; - - int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; - - for (int32_t i = 0; i < nOutputCols; ++i) { - SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i); - SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i); - -// char *buf = pDst->interResultBuf; - memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes); -// pDst->interResultBuf = buf; // restore the allocated buffer - - // copy the result info struct -// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes); - - // copy the output buffer data from src to dst, the position info keep unchanged - tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId); - char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); - - tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId); - char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage); - size_t s = pRuntimeEnv->pQuery->pExpr1[i].bytes; - - memcpy(dstBuf, srcBuf, s); + pResultRow->win = TSWINDOW_INITIALIZER; } } @@ -310,7 +154,7 @@ SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRo return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); } -size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) { +size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); } @@ -384,18 +228,4 @@ void* destroyResultRowPool(SResultRowPool* p) { tfree(p); return NULL; -} - -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { - if (!pRuntimeEnv->stableQuery) { - return 0; // for simple table query, the uid is always set to be 0; - } - - SQuery* pQuery = pRuntimeEnv->pQuery; - if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) { - return 0; - } - - STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); - return id->uid; } \ No newline at end of file diff --git a/src/query/src/sql.c b/src/query/src/sql.c index d4e2b7f5f44677665d26f824506312b943060c40..7c10a4ce6b568940e05dfb82763f4921aea78a8d 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -97,26 +97,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 274 +#define YYNOCODE 278 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - int yy46; - tSQLExpr* yy64; - tVariant yy134; - SCreateAcctSQL yy149; - SArray* yy165; - int64_t yy207; - SLimitVal yy216; - TAOS_FIELD yy223; - SSubclauseInfo* yy231; - SCreateDBInfo yy268; - tSQLExprList* yy290; - SQuerySQL* yy414; - SCreateTableSQL* yy470; - SIntervalVal yy532; + SQuerySQL* yy4; + SSubclauseInfo* yy13; + int yy70; + SCreatedTableInfo yy84; + SIntervalVal yy222; + TAOS_FIELD yy363; + tSQLExprList* yy382; + int64_t yy387; + SArray* yy403; + SLimitVal yy404; + SCreateTableSQL* yy436; + SCreateAcctSQL yy463; + SCreateDBInfo yy478; + tVariant yy488; + tSQLExpr* yy522; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -126,17 +127,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 253 -#define YYNRULE 233 -#define YYNTOKEN 207 -#define YY_MAX_SHIFT 252 -#define YY_MIN_SHIFTREDUCE 420 -#define YY_MAX_SHIFTREDUCE 652 -#define YY_ERROR_ACTION 653 -#define YY_ACCEPT_ACTION 654 -#define YY_NO_ACTION 655 -#define YY_MIN_REDUCE 656 -#define YY_MAX_REDUCE 888 +#define YYNSTATE 258 +#define YYNRULE 239 +#define YYNTOKEN 208 +#define YY_MAX_SHIFT 257 +#define YY_MIN_SHIFTREDUCE 430 +#define YY_MAX_SHIFTREDUCE 668 +#define YY_ERROR_ACTION 669 +#define YY_ACCEPT_ACTION 670 +#define YY_NO_ACTION 671 +#define YY_MIN_REDUCE 672 +#define YY_MAX_REDUCE 910 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -202,224 +203,228 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (571) +#define YY_ACTTAB_COUNT (585) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 108, 463, 141, 11, 654, 252, 802, 463, 140, 464, - /* 10 */ 162, 165, 876, 35, 36, 464, 37, 38, 159, 250, - /* 20 */ 170, 29, 141, 463, 206, 41, 39, 43, 40, 173, - /* 30 */ 780, 464, 875, 34, 33, 145, 141, 32, 31, 30, - /* 40 */ 35, 36, 791, 37, 38, 164, 876, 170, 29, 780, - /* 50 */ 21, 206, 41, 39, 43, 40, 191, 829, 799, 201, - /* 60 */ 34, 33, 21, 21, 32, 31, 30, 421, 422, 423, - /* 70 */ 424, 425, 426, 427, 428, 429, 430, 431, 432, 251, - /* 80 */ 35, 36, 181, 37, 38, 532, 776, 170, 29, 238, - /* 90 */ 246, 206, 41, 39, 43, 40, 174, 175, 777, 777, - /* 100 */ 34, 33, 872, 56, 32, 31, 30, 176, 871, 36, - /* 110 */ 780, 37, 38, 227, 226, 170, 29, 791, 17, 206, - /* 120 */ 41, 39, 43, 40, 108, 26, 870, 606, 34, 33, - /* 130 */ 78, 160, 32, 31, 30, 238, 157, 16, 218, 245, - /* 140 */ 244, 217, 216, 215, 243, 214, 242, 241, 240, 213, - /* 150 */ 239, 755, 103, 743, 744, 745, 746, 747, 748, 749, - /* 160 */ 750, 751, 752, 753, 754, 756, 37, 38, 229, 177, - /* 170 */ 170, 29, 224, 223, 206, 41, 39, 43, 40, 203, - /* 180 */ 62, 60, 8, 34, 33, 63, 118, 32, 31, 30, - /* 190 */ 169, 619, 27, 12, 610, 184, 613, 158, 616, 778, - /* 200 */ 169, 619, 188, 187, 610, 194, 613, 108, 616, 153, - /* 210 */ 169, 619, 561, 108, 610, 154, 613, 18, 616, 90, - /* 220 */ 89, 148, 166, 167, 34, 33, 205, 143, 32, 31, - /* 230 */ 30, 697, 166, 167, 131, 144, 564, 41, 39, 43, - /* 240 */ 40, 706, 166, 167, 131, 34, 33, 146, 17, 32, - /* 250 */ 31, 30, 32, 31, 30, 26, 16, 207, 245, 244, - /* 260 */ 21, 587, 588, 243, 828, 242, 241, 240, 698, 239, - /* 270 */ 61, 131, 76, 80, 147, 190, 102, 151, 85, 88, - /* 280 */ 79, 760, 156, 26, 758, 759, 82, 21, 42, 761, - /* 290 */ 556, 763, 764, 762, 225, 765, 777, 193, 42, 618, - /* 300 */ 249, 248, 96, 574, 121, 122, 608, 105, 42, 618, - /* 310 */ 70, 66, 69, 578, 617, 168, 579, 46, 152, 618, - /* 320 */ 14, 230, 548, 777, 617, 545, 638, 546, 150, 547, - /* 330 */ 13, 135, 133, 612, 617, 615, 139, 93, 92, 91, - /* 340 */ 620, 611, 609, 614, 13, 47, 538, 622, 50, 552, - /* 350 */ 46, 553, 537, 178, 179, 3, 22, 211, 75, 74, - /* 360 */ 149, 22, 10, 9, 48, 51, 142, 550, 885, 551, - /* 370 */ 87, 86, 101, 99, 779, 839, 838, 171, 835, 834, - /* 380 */ 172, 801, 771, 228, 806, 793, 808, 104, 821, 119, - /* 390 */ 820, 117, 120, 708, 212, 137, 24, 221, 705, 222, - /* 400 */ 26, 192, 100, 884, 72, 883, 881, 123, 726, 25, - /* 410 */ 573, 23, 138, 695, 49, 81, 693, 83, 84, 691, - /* 420 */ 790, 690, 195, 161, 199, 549, 57, 52, 180, 132, - /* 430 */ 688, 687, 686, 685, 684, 134, 682, 109, 680, 678, - /* 440 */ 44, 676, 674, 136, 204, 202, 58, 822, 200, 198, - /* 450 */ 196, 220, 77, 28, 231, 232, 233, 235, 652, 234, - /* 460 */ 236, 237, 247, 209, 183, 53, 651, 182, 185, 186, - /* 470 */ 64, 67, 155, 650, 643, 189, 193, 689, 558, 94, - /* 480 */ 683, 675, 126, 125, 727, 129, 124, 127, 128, 95, - /* 490 */ 130, 1, 114, 110, 111, 775, 2, 55, 59, 116, - /* 500 */ 112, 113, 115, 575, 106, 163, 197, 5, 580, 107, - /* 510 */ 6, 65, 621, 19, 4, 20, 15, 208, 623, 7, - /* 520 */ 210, 504, 500, 498, 497, 496, 493, 467, 219, 68, - /* 530 */ 45, 71, 73, 22, 534, 533, 531, 488, 54, 486, - /* 540 */ 478, 484, 480, 482, 476, 474, 505, 503, 502, 501, - /* 550 */ 499, 495, 494, 46, 465, 436, 434, 656, 655, 655, - /* 560 */ 655, 655, 655, 655, 655, 655, 655, 655, 655, 97, - /* 570 */ 98, + /* 0 */ 143, 473, 143, 23, 670, 257, 165, 545, 824, 474, + /* 10 */ 897, 168, 898, 37, 38, 12, 39, 40, 813, 23, + /* 20 */ 173, 31, 473, 473, 209, 43, 41, 45, 42, 802, + /* 30 */ 474, 474, 163, 36, 35, 231, 230, 34, 33, 32, + /* 40 */ 37, 38, 798, 39, 40, 813, 105, 173, 31, 162, + /* 50 */ 255, 209, 43, 41, 45, 42, 176, 178, 799, 194, + /* 60 */ 36, 35, 233, 821, 34, 33, 32, 431, 432, 433, + /* 70 */ 434, 435, 436, 437, 438, 439, 440, 441, 442, 256, + /* 80 */ 802, 143, 184, 63, 179, 37, 38, 224, 39, 40, + /* 90 */ 167, 898, 173, 31, 800, 29, 209, 43, 41, 45, + /* 100 */ 42, 110, 197, 791, 57, 36, 35, 251, 210, 34, + /* 110 */ 33, 32, 110, 17, 222, 250, 249, 221, 220, 219, + /* 120 */ 248, 218, 247, 246, 245, 217, 244, 243, 622, 772, + /* 130 */ 802, 760, 761, 762, 763, 764, 765, 766, 767, 768, + /* 140 */ 769, 770, 771, 773, 774, 242, 38, 180, 39, 40, + /* 150 */ 228, 227, 173, 31, 110, 18, 209, 43, 41, 45, + /* 160 */ 42, 851, 28, 204, 110, 36, 35, 23, 187, 34, + /* 170 */ 33, 32, 850, 39, 40, 191, 190, 173, 31, 224, + /* 180 */ 624, 209, 43, 41, 45, 42, 34, 33, 32, 9, + /* 190 */ 36, 35, 65, 120, 34, 33, 32, 172, 635, 638, + /* 200 */ 66, 626, 104, 629, 177, 632, 799, 172, 635, 28, + /* 210 */ 13, 626, 206, 629, 61, 632, 625, 172, 635, 142, + /* 220 */ 572, 626, 23, 629, 62, 632, 155, 196, 147, 169, + /* 230 */ 170, 793, 156, 208, 603, 604, 92, 91, 150, 169, + /* 240 */ 170, 894, 713, 580, 17, 133, 250, 249, 893, 169, + /* 250 */ 170, 248, 64, 247, 246, 245, 892, 244, 243, 229, + /* 260 */ 778, 799, 80, 776, 777, 590, 18, 242, 779, 107, + /* 270 */ 781, 782, 780, 28, 783, 784, 43, 41, 45, 42, + /* 280 */ 159, 790, 22, 792, 36, 35, 160, 171, 34, 33, + /* 290 */ 32, 722, 564, 193, 133, 561, 44, 562, 52, 563, + /* 300 */ 158, 254, 253, 98, 36, 35, 44, 634, 34, 33, + /* 310 */ 32, 23, 145, 3, 124, 53, 44, 634, 146, 72, + /* 320 */ 68, 71, 633, 181, 182, 148, 714, 634, 4, 133, + /* 330 */ 78, 82, 633, 137, 135, 149, 87, 90, 81, 95, + /* 340 */ 94, 93, 633, 153, 84, 594, 577, 154, 234, 48, + /* 350 */ 799, 19, 49, 152, 595, 654, 636, 141, 15, 14, + /* 360 */ 14, 628, 627, 631, 630, 553, 212, 151, 554, 24, + /* 370 */ 24, 50, 48, 77, 76, 11, 10, 568, 566, 569, + /* 380 */ 567, 89, 88, 103, 101, 907, 144, 801, 861, 860, + /* 390 */ 174, 857, 856, 175, 823, 232, 565, 828, 830, 106, + /* 400 */ 843, 815, 842, 121, 122, 28, 119, 123, 195, 724, + /* 410 */ 216, 139, 26, 225, 721, 226, 906, 74, 102, 905, + /* 420 */ 903, 125, 742, 27, 25, 140, 711, 83, 589, 709, + /* 430 */ 85, 86, 707, 706, 183, 134, 704, 703, 702, 701, + /* 440 */ 198, 700, 136, 698, 696, 694, 692, 690, 138, 164, + /* 450 */ 58, 54, 59, 202, 51, 844, 46, 812, 207, 205, + /* 460 */ 203, 201, 199, 30, 79, 235, 236, 237, 238, 239, + /* 470 */ 240, 161, 214, 241, 252, 215, 668, 186, 185, 157, + /* 480 */ 69, 667, 188, 189, 666, 659, 192, 196, 166, 574, + /* 490 */ 705, 591, 56, 96, 132, 743, 126, 128, 127, 129, + /* 500 */ 130, 699, 111, 131, 1, 97, 116, 112, 113, 114, + /* 510 */ 691, 797, 60, 117, 115, 118, 2, 20, 108, 200, + /* 520 */ 6, 596, 109, 5, 7, 637, 21, 8, 211, 16, + /* 530 */ 213, 639, 67, 65, 514, 510, 508, 507, 506, 503, + /* 540 */ 477, 223, 70, 47, 73, 75, 24, 547, 546, 544, + /* 550 */ 55, 498, 496, 488, 494, 490, 492, 486, 484, 516, + /* 560 */ 515, 513, 512, 511, 509, 505, 504, 48, 475, 446, + /* 570 */ 444, 672, 671, 671, 671, 671, 671, 671, 671, 671, + /* 580 */ 671, 671, 671, 99, 100, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 211, 1, 262, 262, 208, 209, 211, 1, 262, 9, - /* 10 */ 228, 271, 272, 13, 14, 9, 16, 17, 210, 211, - /* 20 */ 20, 21, 262, 1, 24, 25, 26, 27, 28, 228, - /* 30 */ 248, 9, 272, 33, 34, 262, 262, 37, 38, 39, - /* 40 */ 13, 14, 246, 16, 17, 271, 272, 20, 21, 248, - /* 50 */ 211, 24, 25, 26, 27, 28, 260, 268, 263, 270, - /* 60 */ 33, 34, 211, 211, 37, 38, 39, 45, 46, 47, + /* 0 */ 266, 1, 266, 212, 209, 210, 229, 5, 212, 9, + /* 10 */ 276, 275, 276, 13, 14, 266, 16, 17, 250, 212, + /* 20 */ 20, 21, 1, 1, 24, 25, 26, 27, 28, 252, + /* 30 */ 9, 9, 264, 33, 34, 33, 34, 37, 38, 39, + /* 40 */ 13, 14, 251, 16, 17, 250, 212, 20, 21, 211, + /* 50 */ 212, 24, 25, 26, 27, 28, 249, 229, 251, 264, + /* 60 */ 33, 34, 212, 267, 37, 38, 39, 45, 46, 47, /* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 80 */ 13, 14, 60, 16, 17, 5, 247, 20, 21, 78, - /* 90 */ 228, 24, 25, 26, 27, 28, 245, 245, 247, 247, - /* 100 */ 33, 34, 262, 103, 37, 38, 39, 66, 262, 14, - /* 110 */ 248, 16, 17, 33, 34, 20, 21, 246, 99, 24, - /* 120 */ 25, 26, 27, 28, 211, 106, 262, 100, 33, 34, - /* 130 */ 73, 260, 37, 38, 39, 78, 262, 85, 86, 87, - /* 140 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - /* 150 */ 98, 227, 211, 229, 230, 231, 232, 233, 234, 235, - /* 160 */ 236, 237, 238, 239, 240, 241, 16, 17, 211, 128, - /* 170 */ 20, 21, 131, 132, 24, 25, 26, 27, 28, 266, - /* 180 */ 249, 268, 99, 33, 34, 102, 103, 37, 38, 39, - /* 190 */ 1, 2, 261, 44, 5, 127, 7, 262, 9, 242, - /* 200 */ 1, 2, 134, 135, 5, 264, 7, 211, 9, 60, - /* 210 */ 1, 2, 104, 211, 5, 66, 7, 109, 9, 70, - /* 220 */ 71, 72, 33, 34, 33, 34, 37, 262, 37, 38, - /* 230 */ 39, 215, 33, 34, 218, 262, 37, 25, 26, 27, - /* 240 */ 28, 215, 33, 34, 218, 33, 34, 262, 99, 37, - /* 250 */ 38, 39, 37, 38, 39, 106, 85, 15, 87, 88, - /* 260 */ 211, 116, 117, 92, 268, 94, 95, 96, 215, 98, - /* 270 */ 268, 218, 61, 62, 262, 126, 99, 262, 67, 68, - /* 280 */ 69, 227, 133, 106, 230, 231, 75, 211, 99, 235, - /* 290 */ 100, 237, 238, 239, 245, 241, 247, 107, 99, 110, - /* 300 */ 63, 64, 65, 100, 61, 62, 1, 104, 99, 110, - /* 310 */ 67, 68, 69, 100, 125, 59, 100, 104, 262, 110, - /* 320 */ 104, 245, 2, 247, 125, 5, 100, 7, 262, 9, - /* 330 */ 104, 61, 62, 5, 125, 7, 262, 67, 68, 69, - /* 340 */ 100, 5, 37, 7, 104, 104, 100, 105, 104, 5, - /* 350 */ 104, 7, 100, 33, 34, 99, 104, 100, 129, 130, - /* 360 */ 262, 104, 129, 130, 123, 121, 262, 5, 248, 7, - /* 370 */ 73, 74, 61, 62, 248, 243, 243, 243, 243, 243, - /* 380 */ 243, 211, 244, 243, 211, 246, 211, 211, 269, 211, - /* 390 */ 269, 250, 211, 211, 211, 211, 211, 211, 211, 211, - /* 400 */ 106, 246, 59, 211, 211, 211, 211, 211, 211, 211, - /* 410 */ 110, 211, 211, 211, 122, 211, 211, 211, 211, 211, - /* 420 */ 259, 211, 265, 265, 265, 105, 212, 120, 211, 211, - /* 430 */ 211, 211, 211, 211, 211, 211, 211, 258, 211, 211, - /* 440 */ 119, 211, 211, 211, 114, 118, 212, 212, 113, 112, - /* 450 */ 111, 76, 84, 124, 83, 49, 80, 53, 5, 82, - /* 460 */ 81, 79, 76, 212, 5, 212, 5, 136, 136, 5, - /* 470 */ 216, 216, 212, 5, 86, 127, 107, 212, 100, 213, - /* 480 */ 212, 212, 220, 224, 226, 222, 225, 223, 221, 213, - /* 490 */ 219, 217, 253, 257, 256, 246, 214, 108, 104, 251, - /* 500 */ 255, 254, 252, 100, 99, 1, 99, 115, 100, 99, - /* 510 */ 115, 73, 100, 104, 99, 104, 99, 101, 105, 99, - /* 520 */ 101, 9, 5, 5, 5, 5, 5, 77, 15, 73, - /* 530 */ 16, 130, 130, 104, 5, 5, 100, 5, 99, 5, - /* 540 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 550 */ 5, 5, 5, 104, 77, 59, 58, 0, 273, 273, - /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 21, - /* 570 */ 21, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 580 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 590 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 600 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 610 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 620 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 630 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 640 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 650 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 660 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 670 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 680 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 690 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 700 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 710 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 720 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 730 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 740 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 760 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 770 */ 273, 273, 273, 273, 273, 273, 273, 273, + /* 80 */ 252, 266, 60, 253, 66, 13, 14, 76, 16, 17, + /* 90 */ 275, 276, 20, 21, 244, 265, 24, 25, 26, 27, + /* 100 */ 28, 212, 268, 0, 104, 33, 34, 229, 15, 37, + /* 110 */ 38, 39, 212, 85, 86, 87, 88, 89, 90, 91, + /* 120 */ 92, 93, 94, 95, 96, 97, 98, 99, 101, 228, + /* 130 */ 252, 230, 231, 232, 233, 234, 235, 236, 237, 238, + /* 140 */ 239, 240, 241, 242, 243, 78, 14, 129, 16, 17, + /* 150 */ 132, 133, 20, 21, 212, 100, 24, 25, 26, 27, + /* 160 */ 28, 272, 107, 274, 212, 33, 34, 212, 128, 37, + /* 170 */ 38, 39, 272, 16, 17, 135, 136, 20, 21, 76, + /* 180 */ 1, 24, 25, 26, 27, 28, 37, 38, 39, 100, + /* 190 */ 33, 34, 103, 104, 37, 38, 39, 1, 2, 106, + /* 200 */ 217, 5, 100, 7, 249, 9, 251, 1, 2, 107, + /* 210 */ 44, 5, 270, 7, 272, 9, 37, 1, 2, 266, + /* 220 */ 101, 5, 212, 7, 272, 9, 60, 108, 266, 33, + /* 230 */ 34, 248, 66, 37, 117, 118, 70, 71, 72, 33, + /* 240 */ 34, 266, 216, 37, 85, 219, 87, 88, 266, 33, + /* 250 */ 34, 92, 217, 94, 95, 96, 266, 98, 99, 249, + /* 260 */ 228, 251, 73, 231, 232, 101, 100, 78, 236, 105, + /* 270 */ 238, 239, 240, 107, 242, 243, 25, 26, 27, 28, + /* 280 */ 266, 246, 247, 248, 33, 34, 266, 59, 37, 38, + /* 290 */ 39, 216, 2, 127, 219, 5, 100, 7, 105, 9, + /* 300 */ 134, 63, 64, 65, 33, 34, 100, 111, 37, 38, + /* 310 */ 39, 212, 266, 61, 62, 122, 100, 111, 266, 67, + /* 320 */ 68, 69, 126, 33, 34, 266, 216, 111, 100, 219, + /* 330 */ 61, 62, 126, 61, 62, 266, 67, 68, 69, 67, + /* 340 */ 68, 69, 126, 266, 75, 101, 105, 266, 249, 105, + /* 350 */ 251, 110, 105, 266, 101, 101, 101, 266, 105, 105, + /* 360 */ 105, 5, 5, 7, 7, 101, 101, 266, 101, 105, + /* 370 */ 105, 124, 105, 130, 131, 130, 131, 5, 5, 7, + /* 380 */ 7, 73, 74, 61, 62, 252, 266, 252, 245, 245, + /* 390 */ 245, 245, 245, 245, 212, 245, 106, 212, 212, 212, + /* 400 */ 273, 250, 273, 212, 212, 107, 254, 212, 250, 212, + /* 410 */ 212, 212, 212, 212, 212, 212, 212, 212, 59, 212, + /* 420 */ 212, 212, 212, 212, 212, 212, 212, 212, 111, 212, + /* 430 */ 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, + /* 440 */ 269, 212, 212, 212, 212, 212, 212, 212, 212, 269, + /* 450 */ 213, 121, 213, 269, 123, 213, 120, 263, 115, 119, + /* 460 */ 114, 113, 112, 125, 84, 83, 49, 80, 82, 53, + /* 470 */ 81, 213, 213, 79, 76, 213, 5, 5, 137, 213, + /* 480 */ 217, 5, 137, 5, 5, 86, 128, 108, 1, 101, + /* 490 */ 213, 101, 109, 214, 220, 227, 226, 221, 225, 224, + /* 500 */ 222, 213, 262, 223, 218, 214, 257, 261, 260, 259, + /* 510 */ 213, 250, 105, 256, 258, 255, 215, 105, 100, 100, + /* 520 */ 116, 101, 100, 100, 116, 101, 105, 100, 102, 100, + /* 530 */ 102, 106, 73, 103, 9, 5, 5, 5, 5, 5, + /* 540 */ 77, 15, 73, 16, 131, 131, 105, 5, 5, 101, + /* 550 */ 100, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 560 */ 5, 5, 5, 5, 5, 5, 5, 105, 77, 59, + /* 570 */ 58, 0, 277, 277, 277, 277, 277, 277, 277, 277, + /* 580 */ 277, 277, 277, 21, 21, 277, 277, 277, 277, 277, + /* 590 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 600 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 610 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 620 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 630 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 640 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 650 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 660 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 670 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 680 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 690 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 700 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 710 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 720 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 730 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 740 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 750 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 760 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 770 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 780 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 790 */ 277, 277, 277, }; -#define YY_SHIFT_COUNT (252) +#define YY_SHIFT_COUNT (257) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (557) +#define YY_SHIFT_MAX (571) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 149, 52, 171, 189, 209, 6, 6, 6, 6, 6, - /* 10 */ 6, 0, 22, 209, 320, 320, 320, 19, 6, 6, - /* 20 */ 6, 6, 6, 57, 11, 11, 571, 199, 209, 209, - /* 30 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, - /* 40 */ 209, 209, 209, 209, 209, 320, 320, 80, 80, 80, - /* 50 */ 80, 80, 80, 83, 80, 177, 6, 6, 6, 6, - /* 60 */ 145, 145, 108, 6, 6, 6, 6, 6, 6, 6, - /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 100 */ 6, 6, 294, 343, 343, 300, 300, 300, 343, 307, - /* 110 */ 292, 321, 330, 327, 335, 337, 339, 329, 294, 343, - /* 120 */ 343, 375, 375, 343, 368, 371, 406, 376, 377, 404, - /* 130 */ 379, 382, 343, 386, 343, 386, 343, 571, 571, 27, - /* 140 */ 67, 67, 67, 95, 150, 212, 212, 212, 211, 191, - /* 150 */ 191, 191, 191, 243, 270, 41, 68, 215, 215, 237, - /* 160 */ 190, 203, 213, 216, 226, 240, 328, 336, 305, 256, - /* 170 */ 242, 241, 244, 246, 252, 257, 229, 233, 344, 362, - /* 180 */ 297, 311, 453, 331, 459, 461, 332, 464, 468, 388, - /* 190 */ 348, 369, 378, 389, 394, 403, 405, 504, 407, 408, - /* 200 */ 410, 409, 392, 411, 395, 412, 415, 413, 417, 416, - /* 210 */ 420, 419, 438, 512, 517, 518, 519, 520, 521, 450, - /* 220 */ 513, 456, 514, 401, 402, 429, 529, 530, 436, 439, - /* 230 */ 429, 532, 534, 535, 536, 537, 538, 539, 540, 541, - /* 240 */ 542, 543, 544, 545, 546, 547, 449, 477, 548, 549, - /* 250 */ 496, 498, 557, + /* 0 */ 166, 28, 159, 11, 196, 216, 21, 21, 21, 21, + /* 10 */ 21, 21, 0, 22, 216, 290, 290, 290, 55, 21, + /* 20 */ 21, 21, 103, 21, 21, 189, 67, 67, 585, 206, + /* 30 */ 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, + /* 40 */ 216, 216, 216, 216, 216, 216, 216, 290, 290, 2, + /* 50 */ 2, 2, 2, 2, 2, 2, 102, 21, 21, 21, + /* 60 */ 21, 117, 117, 241, 21, 21, 21, 21, 21, 21, + /* 70 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + /* 80 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + /* 90 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + /* 100 */ 21, 21, 21, 21, 298, 359, 359, 317, 317, 317, + /* 110 */ 359, 330, 331, 336, 343, 340, 346, 348, 350, 338, + /* 120 */ 298, 359, 359, 359, 11, 359, 380, 382, 417, 387, + /* 130 */ 386, 416, 389, 394, 359, 398, 359, 398, 359, 585, + /* 140 */ 585, 27, 72, 72, 72, 132, 157, 251, 251, 251, + /* 150 */ 269, 271, 271, 271, 271, 252, 272, 18, 40, 149, + /* 160 */ 149, 89, 238, 119, 164, 244, 253, 254, 255, 356, + /* 170 */ 357, 179, 228, 93, 247, 193, 264, 265, 267, 243, + /* 180 */ 245, 372, 373, 308, 322, 471, 341, 472, 476, 345, + /* 190 */ 478, 479, 399, 358, 379, 388, 383, 407, 390, 418, + /* 200 */ 487, 419, 420, 422, 412, 404, 421, 408, 424, 423, + /* 210 */ 425, 427, 426, 429, 428, 430, 459, 525, 530, 531, + /* 220 */ 532, 533, 534, 463, 526, 469, 527, 413, 414, 441, + /* 230 */ 542, 543, 448, 450, 441, 546, 547, 548, 549, 550, + /* 240 */ 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, + /* 250 */ 561, 462, 491, 562, 563, 510, 512, 571, }; -#define YY_REDUCE_COUNT (138) -#define YY_REDUCE_MIN (-260) -#define YY_REDUCE_MAX (282) +#define YY_REDUCE_COUNT (140) +#define YY_REDUCE_MIN (-266) +#define YY_REDUCE_MAX (301) static const short yy_reduce_ofst[] = { - /* 0 */ -204, -76, 54, -260, -226, -211, -87, -149, -148, 49, - /* 10 */ 76, -205, -192, -240, -218, -199, -138, -129, -59, -4, - /* 20 */ 2, -43, -161, 16, 26, 53, -69, -259, -254, -227, - /* 30 */ -160, -154, -136, -126, -65, -35, -27, -15, 12, 15, - /* 40 */ 56, 66, 74, 98, 104, 120, 126, 132, 133, 134, - /* 50 */ 135, 136, 137, 138, 140, 139, 170, 173, 175, 176, - /* 60 */ 119, 121, 141, 178, 181, 182, 183, 184, 185, 186, - /* 70 */ 187, 188, 192, 193, 194, 195, 196, 197, 198, 200, - /* 80 */ 201, 202, 204, 205, 206, 207, 208, 210, 217, 218, - /* 90 */ 219, 220, 221, 222, 223, 224, 225, 227, 228, 230, - /* 100 */ 231, 232, 155, 214, 234, 157, 158, 159, 235, 161, - /* 110 */ 179, 236, 238, 245, 247, 239, 250, 248, 249, 251, - /* 120 */ 253, 254, 255, 260, 258, 261, 259, 262, 264, 267, - /* 130 */ 263, 271, 265, 266, 268, 276, 269, 274, 282, + /* 0 */ -205, -99, 32, 35, -264, -185, -111, -58, -193, -45, + /* 10 */ 10, 99, -204, -162, -266, -223, -172, -122, -232, -166, + /* 20 */ -100, -48, -17, -150, -209, 26, 75, 110, -170, -251, + /* 30 */ -47, -38, -25, -18, -10, 14, 20, 46, 52, 59, + /* 40 */ 69, 77, 81, 87, 91, 101, 120, 133, 135, 143, + /* 50 */ 144, 145, 146, 147, 148, 150, 151, 182, 185, 186, + /* 60 */ 187, 127, 129, 152, 191, 192, 195, 197, 198, 199, + /* 70 */ 200, 201, 202, 203, 204, 205, 207, 208, 209, 210, + /* 80 */ 211, 212, 213, 214, 215, 217, 218, 219, 220, 221, + /* 90 */ 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, + /* 100 */ 233, 234, 235, 236, 158, 237, 239, 171, 180, 184, + /* 110 */ 242, 194, 240, 246, 248, 250, 256, 249, 257, 260, + /* 120 */ 261, 258, 259, 262, 263, 266, 268, 270, 273, 276, + /* 130 */ 275, 278, 280, 274, 277, 279, 288, 291, 297, 286, + /* 140 */ 301, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 653, 707, 696, 878, 878, 653, 653, 653, 653, 653, - /* 10 */ 653, 803, 671, 878, 653, 653, 653, 653, 653, 653, - /* 20 */ 653, 653, 653, 709, 709, 709, 798, 653, 653, 653, - /* 30 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 40 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 50 */ 653, 653, 653, 653, 653, 653, 653, 805, 807, 653, - /* 60 */ 825, 825, 796, 653, 653, 653, 653, 653, 653, 653, - /* 70 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 80 */ 653, 694, 653, 692, 653, 653, 653, 653, 653, 653, - /* 90 */ 653, 653, 653, 653, 653, 653, 681, 653, 653, 653, - /* 100 */ 653, 653, 653, 673, 673, 653, 653, 653, 673, 832, - /* 110 */ 836, 830, 818, 826, 817, 813, 812, 840, 653, 673, - /* 120 */ 673, 704, 704, 673, 725, 723, 721, 713, 719, 715, - /* 130 */ 717, 711, 673, 702, 673, 702, 673, 742, 757, 653, - /* 140 */ 841, 877, 831, 867, 866, 873, 865, 864, 653, 860, - /* 150 */ 861, 863, 862, 653, 653, 653, 653, 869, 868, 653, - /* 160 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 843, - /* 170 */ 653, 837, 833, 653, 653, 653, 653, 653, 653, 653, - /* 180 */ 653, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 190 */ 653, 795, 653, 653, 804, 653, 653, 653, 653, 653, - /* 200 */ 653, 827, 653, 819, 653, 653, 653, 653, 653, 653, - /* 210 */ 653, 772, 653, 653, 653, 653, 653, 653, 653, 653, - /* 220 */ 653, 653, 653, 653, 653, 882, 653, 653, 653, 766, - /* 230 */ 880, 653, 653, 653, 653, 653, 653, 653, 653, 653, - /* 240 */ 653, 653, 653, 653, 653, 653, 728, 653, 679, 677, - /* 250 */ 653, 669, 653, + /* 0 */ 669, 723, 712, 720, 900, 900, 669, 669, 669, 669, + /* 10 */ 669, 669, 825, 687, 900, 669, 669, 669, 669, 669, + /* 20 */ 669, 669, 720, 669, 669, 725, 725, 725, 820, 669, + /* 30 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 40 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 50 */ 669, 669, 669, 669, 669, 669, 669, 669, 827, 829, + /* 60 */ 669, 847, 847, 818, 669, 669, 669, 669, 669, 669, + /* 70 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 80 */ 669, 669, 669, 710, 669, 708, 669, 669, 669, 669, + /* 90 */ 669, 669, 669, 669, 669, 669, 669, 669, 697, 669, + /* 100 */ 669, 669, 669, 669, 669, 689, 689, 669, 669, 669, + /* 110 */ 689, 854, 858, 852, 840, 848, 839, 835, 834, 862, + /* 120 */ 669, 689, 689, 689, 720, 689, 741, 739, 737, 729, + /* 130 */ 735, 731, 733, 727, 689, 718, 689, 718, 689, 759, + /* 140 */ 775, 669, 863, 899, 853, 889, 888, 895, 887, 886, + /* 150 */ 669, 882, 883, 885, 884, 669, 669, 669, 669, 891, + /* 160 */ 890, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 170 */ 669, 669, 865, 669, 859, 855, 669, 669, 669, 669, + /* 180 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 190 */ 669, 669, 669, 669, 817, 669, 669, 826, 669, 669, + /* 200 */ 669, 669, 669, 669, 849, 669, 841, 669, 669, 669, + /* 210 */ 669, 669, 794, 669, 669, 669, 669, 669, 669, 669, + /* 220 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 904, + /* 230 */ 669, 669, 669, 785, 902, 669, 669, 669, 669, 669, + /* 240 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 250 */ 669, 744, 669, 695, 693, 669, 685, 669, }; /********** End of lemon-generated parsing tables *****************************/ @@ -538,6 +543,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* COMP => nothing */ 0, /* PRECISION => nothing */ 0, /* UPDATE => nothing */ + 0, /* CACHELAST => nothing */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* TAGS => nothing */ @@ -831,180 +837,184 @@ static const char *const yyTokenName[] = { /* 96 */ "COMP", /* 97 */ "PRECISION", /* 98 */ "UPDATE", - /* 99 */ "LP", - /* 100 */ "RP", - /* 101 */ "TAGS", - /* 102 */ "USING", - /* 103 */ "AS", - /* 104 */ "COMMA", - /* 105 */ "NULL", - /* 106 */ "SELECT", - /* 107 */ "UNION", - /* 108 */ "ALL", - /* 109 */ "FROM", - /* 110 */ "VARIABLE", - /* 111 */ "INTERVAL", - /* 112 */ "FILL", - /* 113 */ "SLIDING", - /* 114 */ "ORDER", - /* 115 */ "BY", - /* 116 */ "ASC", - /* 117 */ "DESC", - /* 118 */ "GROUP", - /* 119 */ "HAVING", - /* 120 */ "LIMIT", - /* 121 */ "OFFSET", - /* 122 */ "SLIMIT", - /* 123 */ "SOFFSET", - /* 124 */ "WHERE", - /* 125 */ "NOW", - /* 126 */ "RESET", - /* 127 */ "QUERY", - /* 128 */ "ADD", - /* 129 */ "COLUMN", - /* 130 */ "TAG", - /* 131 */ "CHANGE", - /* 132 */ "SET", - /* 133 */ "KILL", - /* 134 */ "CONNECTION", - /* 135 */ "STREAM", - /* 136 */ "COLON", - /* 137 */ "ABORT", - /* 138 */ "AFTER", - /* 139 */ "ATTACH", - /* 140 */ "BEFORE", - /* 141 */ "BEGIN", - /* 142 */ "CASCADE", - /* 143 */ "CLUSTER", - /* 144 */ "CONFLICT", - /* 145 */ "COPY", - /* 146 */ "DEFERRED", - /* 147 */ "DELIMITERS", - /* 148 */ "DETACH", - /* 149 */ "EACH", - /* 150 */ "END", - /* 151 */ "EXPLAIN", - /* 152 */ "FAIL", - /* 153 */ "FOR", - /* 154 */ "IGNORE", - /* 155 */ "IMMEDIATE", - /* 156 */ "INITIALLY", - /* 157 */ "INSTEAD", - /* 158 */ "MATCH", - /* 159 */ "KEY", - /* 160 */ "OF", - /* 161 */ "RAISE", - /* 162 */ "REPLACE", - /* 163 */ "RESTRICT", - /* 164 */ "ROW", - /* 165 */ "STATEMENT", - /* 166 */ "TRIGGER", - /* 167 */ "VIEW", - /* 168 */ "COUNT", - /* 169 */ "SUM", - /* 170 */ "AVG", - /* 171 */ "MIN", - /* 172 */ "MAX", - /* 173 */ "FIRST", - /* 174 */ "LAST", - /* 175 */ "TOP", - /* 176 */ "BOTTOM", - /* 177 */ "STDDEV", - /* 178 */ "PERCENTILE", - /* 179 */ "APERCENTILE", - /* 180 */ "LEASTSQUARES", - /* 181 */ "HISTOGRAM", - /* 182 */ "DIFF", - /* 183 */ "SPREAD", - /* 184 */ "TWA", - /* 185 */ "INTERP", - /* 186 */ "LAST_ROW", - /* 187 */ "RATE", - /* 188 */ "IRATE", - /* 189 */ "SUM_RATE", - /* 190 */ "SUM_IRATE", - /* 191 */ "AVG_RATE", - /* 192 */ "AVG_IRATE", - /* 193 */ "TBID", - /* 194 */ "SEMI", - /* 195 */ "NONE", - /* 196 */ "PREV", - /* 197 */ "LINEAR", - /* 198 */ "IMPORT", - /* 199 */ "METRIC", - /* 200 */ "TBNAME", - /* 201 */ "JOIN", - /* 202 */ "METRICS", - /* 203 */ "STABLE", - /* 204 */ "INSERT", - /* 205 */ "INTO", - /* 206 */ "VALUES", - /* 207 */ "error", - /* 208 */ "program", - /* 209 */ "cmd", - /* 210 */ "dbPrefix", - /* 211 */ "ids", - /* 212 */ "cpxName", - /* 213 */ "ifexists", - /* 214 */ "alter_db_optr", - /* 215 */ "acct_optr", - /* 216 */ "ifnotexists", - /* 217 */ "db_optr", - /* 218 */ "pps", - /* 219 */ "tseries", - /* 220 */ "dbs", - /* 221 */ "streams", - /* 222 */ "storage", - /* 223 */ "qtime", - /* 224 */ "users", - /* 225 */ "conns", - /* 226 */ "state", - /* 227 */ "keep", - /* 228 */ "tagitemlist", - /* 229 */ "cache", - /* 230 */ "replica", - /* 231 */ "quorum", - /* 232 */ "days", - /* 233 */ "minrows", - /* 234 */ "maxrows", - /* 235 */ "blocks", - /* 236 */ "ctime", - /* 237 */ "wal", - /* 238 */ "fsync", - /* 239 */ "comp", - /* 240 */ "prec", - /* 241 */ "update", - /* 242 */ "typename", - /* 243 */ "signed", - /* 244 */ "create_table_args", - /* 245 */ "columnlist", - /* 246 */ "select", - /* 247 */ "column", - /* 248 */ "tagitem", - /* 249 */ "selcollist", - /* 250 */ "from", - /* 251 */ "where_opt", - /* 252 */ "interval_opt", - /* 253 */ "fill_opt", - /* 254 */ "sliding_opt", - /* 255 */ "groupby_opt", - /* 256 */ "orderby_opt", - /* 257 */ "having_opt", - /* 258 */ "slimit_opt", - /* 259 */ "limit_opt", - /* 260 */ "union", - /* 261 */ "sclp", - /* 262 */ "expr", - /* 263 */ "as", - /* 264 */ "tablelist", - /* 265 */ "tmvar", - /* 266 */ "sortlist", - /* 267 */ "sortitem", - /* 268 */ "item", - /* 269 */ "sortorder", - /* 270 */ "grouplist", - /* 271 */ "exprlist", - /* 272 */ "expritem", + /* 99 */ "CACHELAST", + /* 100 */ "LP", + /* 101 */ "RP", + /* 102 */ "TAGS", + /* 103 */ "USING", + /* 104 */ "AS", + /* 105 */ "COMMA", + /* 106 */ "NULL", + /* 107 */ "SELECT", + /* 108 */ "UNION", + /* 109 */ "ALL", + /* 110 */ "FROM", + /* 111 */ "VARIABLE", + /* 112 */ "INTERVAL", + /* 113 */ "FILL", + /* 114 */ "SLIDING", + /* 115 */ "ORDER", + /* 116 */ "BY", + /* 117 */ "ASC", + /* 118 */ "DESC", + /* 119 */ "GROUP", + /* 120 */ "HAVING", + /* 121 */ "LIMIT", + /* 122 */ "OFFSET", + /* 123 */ "SLIMIT", + /* 124 */ "SOFFSET", + /* 125 */ "WHERE", + /* 126 */ "NOW", + /* 127 */ "RESET", + /* 128 */ "QUERY", + /* 129 */ "ADD", + /* 130 */ "COLUMN", + /* 131 */ "TAG", + /* 132 */ "CHANGE", + /* 133 */ "SET", + /* 134 */ "KILL", + /* 135 */ "CONNECTION", + /* 136 */ "STREAM", + /* 137 */ "COLON", + /* 138 */ "ABORT", + /* 139 */ "AFTER", + /* 140 */ "ATTACH", + /* 141 */ "BEFORE", + /* 142 */ "BEGIN", + /* 143 */ "CASCADE", + /* 144 */ "CLUSTER", + /* 145 */ "CONFLICT", + /* 146 */ "COPY", + /* 147 */ "DEFERRED", + /* 148 */ "DELIMITERS", + /* 149 */ "DETACH", + /* 150 */ "EACH", + /* 151 */ "END", + /* 152 */ "EXPLAIN", + /* 153 */ "FAIL", + /* 154 */ "FOR", + /* 155 */ "IGNORE", + /* 156 */ "IMMEDIATE", + /* 157 */ "INITIALLY", + /* 158 */ "INSTEAD", + /* 159 */ "MATCH", + /* 160 */ "KEY", + /* 161 */ "OF", + /* 162 */ "RAISE", + /* 163 */ "REPLACE", + /* 164 */ "RESTRICT", + /* 165 */ "ROW", + /* 166 */ "STATEMENT", + /* 167 */ "TRIGGER", + /* 168 */ "VIEW", + /* 169 */ "COUNT", + /* 170 */ "SUM", + /* 171 */ "AVG", + /* 172 */ "MIN", + /* 173 */ "MAX", + /* 174 */ "FIRST", + /* 175 */ "LAST", + /* 176 */ "TOP", + /* 177 */ "BOTTOM", + /* 178 */ "STDDEV", + /* 179 */ "PERCENTILE", + /* 180 */ "APERCENTILE", + /* 181 */ "LEASTSQUARES", + /* 182 */ "HISTOGRAM", + /* 183 */ "DIFF", + /* 184 */ "SPREAD", + /* 185 */ "TWA", + /* 186 */ "INTERP", + /* 187 */ "LAST_ROW", + /* 188 */ "RATE", + /* 189 */ "IRATE", + /* 190 */ "SUM_RATE", + /* 191 */ "SUM_IRATE", + /* 192 */ "AVG_RATE", + /* 193 */ "AVG_IRATE", + /* 194 */ "TBID", + /* 195 */ "SEMI", + /* 196 */ "NONE", + /* 197 */ "PREV", + /* 198 */ "LINEAR", + /* 199 */ "IMPORT", + /* 200 */ "METRIC", + /* 201 */ "TBNAME", + /* 202 */ "JOIN", + /* 203 */ "METRICS", + /* 204 */ "STABLE", + /* 205 */ "INSERT", + /* 206 */ "INTO", + /* 207 */ "VALUES", + /* 208 */ "error", + /* 209 */ "program", + /* 210 */ "cmd", + /* 211 */ "dbPrefix", + /* 212 */ "ids", + /* 213 */ "cpxName", + /* 214 */ "ifexists", + /* 215 */ "alter_db_optr", + /* 216 */ "acct_optr", + /* 217 */ "ifnotexists", + /* 218 */ "db_optr", + /* 219 */ "pps", + /* 220 */ "tseries", + /* 221 */ "dbs", + /* 222 */ "streams", + /* 223 */ "storage", + /* 224 */ "qtime", + /* 225 */ "users", + /* 226 */ "conns", + /* 227 */ "state", + /* 228 */ "keep", + /* 229 */ "tagitemlist", + /* 230 */ "cache", + /* 231 */ "replica", + /* 232 */ "quorum", + /* 233 */ "days", + /* 234 */ "minrows", + /* 235 */ "maxrows", + /* 236 */ "blocks", + /* 237 */ "ctime", + /* 238 */ "wal", + /* 239 */ "fsync", + /* 240 */ "comp", + /* 241 */ "prec", + /* 242 */ "update", + /* 243 */ "cachelast", + /* 244 */ "typename", + /* 245 */ "signed", + /* 246 */ "create_table_args", + /* 247 */ "create_table_list", + /* 248 */ "create_from_stable", + /* 249 */ "columnlist", + /* 250 */ "select", + /* 251 */ "column", + /* 252 */ "tagitem", + /* 253 */ "selcollist", + /* 254 */ "from", + /* 255 */ "where_opt", + /* 256 */ "interval_opt", + /* 257 */ "fill_opt", + /* 258 */ "sliding_opt", + /* 259 */ "groupby_opt", + /* 260 */ "orderby_opt", + /* 261 */ "having_opt", + /* 262 */ "slimit_opt", + /* 263 */ "limit_opt", + /* 264 */ "union", + /* 265 */ "sclp", + /* 266 */ "expr", + /* 267 */ "as", + /* 268 */ "tablelist", + /* 269 */ "tmvar", + /* 270 */ "sortlist", + /* 271 */ "sortitem", + /* 272 */ "item", + /* 273 */ "sortorder", + /* 274 */ "grouplist", + /* 275 */ "exprlist", + /* 276 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1098,153 +1108,159 @@ static const char *const yyRuleName[] = { /* 83 */ "comp ::= COMP INTEGER", /* 84 */ "prec ::= PRECISION STRING", /* 85 */ "update ::= UPDATE INTEGER", - /* 86 */ "db_optr ::=", - /* 87 */ "db_optr ::= db_optr cache", - /* 88 */ "db_optr ::= db_optr replica", - /* 89 */ "db_optr ::= db_optr quorum", - /* 90 */ "db_optr ::= db_optr days", - /* 91 */ "db_optr ::= db_optr minrows", - /* 92 */ "db_optr ::= db_optr maxrows", - /* 93 */ "db_optr ::= db_optr blocks", - /* 94 */ "db_optr ::= db_optr ctime", - /* 95 */ "db_optr ::= db_optr wal", - /* 96 */ "db_optr ::= db_optr fsync", - /* 97 */ "db_optr ::= db_optr comp", - /* 98 */ "db_optr ::= db_optr prec", - /* 99 */ "db_optr ::= db_optr keep", - /* 100 */ "db_optr ::= db_optr update", - /* 101 */ "alter_db_optr ::=", - /* 102 */ "alter_db_optr ::= alter_db_optr replica", - /* 103 */ "alter_db_optr ::= alter_db_optr quorum", - /* 104 */ "alter_db_optr ::= alter_db_optr keep", - /* 105 */ "alter_db_optr ::= alter_db_optr blocks", - /* 106 */ "alter_db_optr ::= alter_db_optr comp", - /* 107 */ "alter_db_optr ::= alter_db_optr wal", - /* 108 */ "alter_db_optr ::= alter_db_optr fsync", - /* 109 */ "alter_db_optr ::= alter_db_optr update", - /* 110 */ "typename ::= ids", - /* 111 */ "typename ::= ids LP signed RP", - /* 112 */ "signed ::= INTEGER", - /* 113 */ "signed ::= PLUS INTEGER", - /* 114 */ "signed ::= MINUS INTEGER", - /* 115 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", - /* 116 */ "create_table_args ::= LP columnlist RP", - /* 117 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", - /* 118 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", - /* 119 */ "create_table_args ::= AS select", - /* 120 */ "columnlist ::= columnlist COMMA column", - /* 121 */ "columnlist ::= column", - /* 122 */ "column ::= ids typename", - /* 123 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 124 */ "tagitemlist ::= tagitem", - /* 125 */ "tagitem ::= INTEGER", - /* 126 */ "tagitem ::= FLOAT", - /* 127 */ "tagitem ::= STRING", - /* 128 */ "tagitem ::= BOOL", - /* 129 */ "tagitem ::= NULL", - /* 130 */ "tagitem ::= MINUS INTEGER", - /* 131 */ "tagitem ::= MINUS FLOAT", - /* 132 */ "tagitem ::= PLUS INTEGER", - /* 133 */ "tagitem ::= PLUS FLOAT", - /* 134 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 135 */ "union ::= select", - /* 136 */ "union ::= LP union RP", - /* 137 */ "union ::= union UNION ALL select", - /* 138 */ "union ::= union UNION ALL LP select RP", - /* 139 */ "cmd ::= union", - /* 140 */ "select ::= SELECT selcollist", - /* 141 */ "sclp ::= selcollist COMMA", - /* 142 */ "sclp ::=", - /* 143 */ "selcollist ::= sclp expr as", - /* 144 */ "selcollist ::= sclp STAR", - /* 145 */ "as ::= AS ids", - /* 146 */ "as ::= ids", - /* 147 */ "as ::=", - /* 148 */ "from ::= FROM tablelist", - /* 149 */ "tablelist ::= ids cpxName", - /* 150 */ "tablelist ::= ids cpxName ids", - /* 151 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 152 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 153 */ "tmvar ::= VARIABLE", - /* 154 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 155 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 156 */ "interval_opt ::=", - /* 157 */ "fill_opt ::=", - /* 158 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 159 */ "fill_opt ::= FILL LP ID RP", - /* 160 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 161 */ "sliding_opt ::=", - /* 162 */ "orderby_opt ::=", - /* 163 */ "orderby_opt ::= ORDER BY sortlist", - /* 164 */ "sortlist ::= sortlist COMMA item sortorder", - /* 165 */ "sortlist ::= item sortorder", - /* 166 */ "item ::= ids cpxName", - /* 167 */ "sortorder ::= ASC", - /* 168 */ "sortorder ::= DESC", - /* 169 */ "sortorder ::=", - /* 170 */ "groupby_opt ::=", - /* 171 */ "groupby_opt ::= GROUP BY grouplist", - /* 172 */ "grouplist ::= grouplist COMMA item", - /* 173 */ "grouplist ::= item", - /* 174 */ "having_opt ::=", - /* 175 */ "having_opt ::= HAVING expr", - /* 176 */ "limit_opt ::=", - /* 177 */ "limit_opt ::= LIMIT signed", - /* 178 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 179 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 180 */ "slimit_opt ::=", - /* 181 */ "slimit_opt ::= SLIMIT signed", - /* 182 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 183 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 184 */ "where_opt ::=", - /* 185 */ "where_opt ::= WHERE expr", - /* 186 */ "expr ::= LP expr RP", - /* 187 */ "expr ::= ID", - /* 188 */ "expr ::= ID DOT ID", - /* 189 */ "expr ::= ID DOT STAR", - /* 190 */ "expr ::= INTEGER", - /* 191 */ "expr ::= MINUS INTEGER", - /* 192 */ "expr ::= PLUS INTEGER", - /* 193 */ "expr ::= FLOAT", - /* 194 */ "expr ::= MINUS FLOAT", - /* 195 */ "expr ::= PLUS FLOAT", - /* 196 */ "expr ::= STRING", - /* 197 */ "expr ::= NOW", - /* 198 */ "expr ::= VARIABLE", - /* 199 */ "expr ::= BOOL", - /* 200 */ "expr ::= ID LP exprlist RP", - /* 201 */ "expr ::= ID LP STAR RP", - /* 202 */ "expr ::= expr IS NULL", - /* 203 */ "expr ::= expr IS NOT NULL", - /* 204 */ "expr ::= expr LT expr", - /* 205 */ "expr ::= expr GT expr", - /* 206 */ "expr ::= expr LE expr", - /* 207 */ "expr ::= expr GE expr", - /* 208 */ "expr ::= expr NE expr", - /* 209 */ "expr ::= expr EQ expr", - /* 210 */ "expr ::= expr AND expr", - /* 211 */ "expr ::= expr OR expr", - /* 212 */ "expr ::= expr PLUS expr", - /* 213 */ "expr ::= expr MINUS expr", - /* 214 */ "expr ::= expr STAR expr", - /* 215 */ "expr ::= expr SLASH expr", - /* 216 */ "expr ::= expr REM expr", - /* 217 */ "expr ::= expr LIKE expr", - /* 218 */ "expr ::= expr IN LP exprlist RP", - /* 219 */ "exprlist ::= exprlist COMMA expritem", - /* 220 */ "exprlist ::= expritem", - /* 221 */ "expritem ::= expr", - /* 222 */ "expritem ::=", - /* 223 */ "cmd ::= RESET QUERY CACHE", - /* 224 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 225 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 226 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 227 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 228 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 229 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 230 */ "cmd ::= KILL CONNECTION INTEGER", - /* 231 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 232 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 86 */ "cachelast ::= CACHELAST INTEGER", + /* 87 */ "db_optr ::=", + /* 88 */ "db_optr ::= db_optr cache", + /* 89 */ "db_optr ::= db_optr replica", + /* 90 */ "db_optr ::= db_optr quorum", + /* 91 */ "db_optr ::= db_optr days", + /* 92 */ "db_optr ::= db_optr minrows", + /* 93 */ "db_optr ::= db_optr maxrows", + /* 94 */ "db_optr ::= db_optr blocks", + /* 95 */ "db_optr ::= db_optr ctime", + /* 96 */ "db_optr ::= db_optr wal", + /* 97 */ "db_optr ::= db_optr fsync", + /* 98 */ "db_optr ::= db_optr comp", + /* 99 */ "db_optr ::= db_optr prec", + /* 100 */ "db_optr ::= db_optr keep", + /* 101 */ "db_optr ::= db_optr update", + /* 102 */ "db_optr ::= db_optr cachelast", + /* 103 */ "alter_db_optr ::=", + /* 104 */ "alter_db_optr ::= alter_db_optr replica", + /* 105 */ "alter_db_optr ::= alter_db_optr quorum", + /* 106 */ "alter_db_optr ::= alter_db_optr keep", + /* 107 */ "alter_db_optr ::= alter_db_optr blocks", + /* 108 */ "alter_db_optr ::= alter_db_optr comp", + /* 109 */ "alter_db_optr ::= alter_db_optr wal", + /* 110 */ "alter_db_optr ::= alter_db_optr fsync", + /* 111 */ "alter_db_optr ::= alter_db_optr update", + /* 112 */ "alter_db_optr ::= alter_db_optr cachelast", + /* 113 */ "typename ::= ids", + /* 114 */ "typename ::= ids LP signed RP", + /* 115 */ "signed ::= INTEGER", + /* 116 */ "signed ::= PLUS INTEGER", + /* 117 */ "signed ::= MINUS INTEGER", + /* 118 */ "cmd ::= CREATE TABLE create_table_args", + /* 119 */ "cmd ::= CREATE TABLE create_table_list", + /* 120 */ "create_table_list ::= create_from_stable", + /* 121 */ "create_table_list ::= create_table_list create_from_stable", + /* 122 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", + /* 123 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", + /* 124 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", + /* 125 */ "create_table_args ::= ifnotexists ids cpxName AS select", + /* 126 */ "columnlist ::= columnlist COMMA column", + /* 127 */ "columnlist ::= column", + /* 128 */ "column ::= ids typename", + /* 129 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 130 */ "tagitemlist ::= tagitem", + /* 131 */ "tagitem ::= INTEGER", + /* 132 */ "tagitem ::= FLOAT", + /* 133 */ "tagitem ::= STRING", + /* 134 */ "tagitem ::= BOOL", + /* 135 */ "tagitem ::= NULL", + /* 136 */ "tagitem ::= MINUS INTEGER", + /* 137 */ "tagitem ::= MINUS FLOAT", + /* 138 */ "tagitem ::= PLUS INTEGER", + /* 139 */ "tagitem ::= PLUS FLOAT", + /* 140 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 141 */ "union ::= select", + /* 142 */ "union ::= LP union RP", + /* 143 */ "union ::= union UNION ALL select", + /* 144 */ "union ::= union UNION ALL LP select RP", + /* 145 */ "cmd ::= union", + /* 146 */ "select ::= SELECT selcollist", + /* 147 */ "sclp ::= selcollist COMMA", + /* 148 */ "sclp ::=", + /* 149 */ "selcollist ::= sclp expr as", + /* 150 */ "selcollist ::= sclp STAR", + /* 151 */ "as ::= AS ids", + /* 152 */ "as ::= ids", + /* 153 */ "as ::=", + /* 154 */ "from ::= FROM tablelist", + /* 155 */ "tablelist ::= ids cpxName", + /* 156 */ "tablelist ::= ids cpxName ids", + /* 157 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 158 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 159 */ "tmvar ::= VARIABLE", + /* 160 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 161 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 162 */ "interval_opt ::=", + /* 163 */ "fill_opt ::=", + /* 164 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 165 */ "fill_opt ::= FILL LP ID RP", + /* 166 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 167 */ "sliding_opt ::=", + /* 168 */ "orderby_opt ::=", + /* 169 */ "orderby_opt ::= ORDER BY sortlist", + /* 170 */ "sortlist ::= sortlist COMMA item sortorder", + /* 171 */ "sortlist ::= item sortorder", + /* 172 */ "item ::= ids cpxName", + /* 173 */ "sortorder ::= ASC", + /* 174 */ "sortorder ::= DESC", + /* 175 */ "sortorder ::=", + /* 176 */ "groupby_opt ::=", + /* 177 */ "groupby_opt ::= GROUP BY grouplist", + /* 178 */ "grouplist ::= grouplist COMMA item", + /* 179 */ "grouplist ::= item", + /* 180 */ "having_opt ::=", + /* 181 */ "having_opt ::= HAVING expr", + /* 182 */ "limit_opt ::=", + /* 183 */ "limit_opt ::= LIMIT signed", + /* 184 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 185 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 186 */ "slimit_opt ::=", + /* 187 */ "slimit_opt ::= SLIMIT signed", + /* 188 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 189 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 190 */ "where_opt ::=", + /* 191 */ "where_opt ::= WHERE expr", + /* 192 */ "expr ::= LP expr RP", + /* 193 */ "expr ::= ID", + /* 194 */ "expr ::= ID DOT ID", + /* 195 */ "expr ::= ID DOT STAR", + /* 196 */ "expr ::= INTEGER", + /* 197 */ "expr ::= MINUS INTEGER", + /* 198 */ "expr ::= PLUS INTEGER", + /* 199 */ "expr ::= FLOAT", + /* 200 */ "expr ::= MINUS FLOAT", + /* 201 */ "expr ::= PLUS FLOAT", + /* 202 */ "expr ::= STRING", + /* 203 */ "expr ::= NOW", + /* 204 */ "expr ::= VARIABLE", + /* 205 */ "expr ::= BOOL", + /* 206 */ "expr ::= ID LP exprlist RP", + /* 207 */ "expr ::= ID LP STAR RP", + /* 208 */ "expr ::= expr IS NULL", + /* 209 */ "expr ::= expr IS NOT NULL", + /* 210 */ "expr ::= expr LT expr", + /* 211 */ "expr ::= expr GT expr", + /* 212 */ "expr ::= expr LE expr", + /* 213 */ "expr ::= expr GE expr", + /* 214 */ "expr ::= expr NE expr", + /* 215 */ "expr ::= expr EQ expr", + /* 216 */ "expr ::= expr AND expr", + /* 217 */ "expr ::= expr OR expr", + /* 218 */ "expr ::= expr PLUS expr", + /* 219 */ "expr ::= expr MINUS expr", + /* 220 */ "expr ::= expr STAR expr", + /* 221 */ "expr ::= expr SLASH expr", + /* 222 */ "expr ::= expr REM expr", + /* 223 */ "expr ::= expr LIKE expr", + /* 224 */ "expr ::= expr IN LP exprlist RP", + /* 225 */ "exprlist ::= exprlist COMMA expritem", + /* 226 */ "exprlist ::= expritem", + /* 227 */ "expritem ::= expr", + /* 228 */ "expritem ::=", + /* 229 */ "cmd ::= RESET QUERY CACHE", + /* 230 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 231 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 232 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 233 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 234 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 235 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 236 */ "cmd ::= KILL CONNECTION INTEGER", + /* 237 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 238 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1365,46 +1381,51 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 227: /* keep */ - case 228: /* tagitemlist */ - case 245: /* columnlist */ - case 253: /* fill_opt */ - case 255: /* groupby_opt */ - case 256: /* orderby_opt */ - case 266: /* sortlist */ - case 270: /* grouplist */ + case 228: /* keep */ + case 229: /* tagitemlist */ + case 249: /* columnlist */ + case 257: /* fill_opt */ + case 259: /* groupby_opt */ + case 260: /* orderby_opt */ + case 270: /* sortlist */ + case 274: /* grouplist */ { -taosArrayDestroy((yypminor->yy165)); +taosArrayDestroy((yypminor->yy403)); } break; - case 246: /* select */ + case 247: /* create_table_list */ { -doDestroyQuerySql((yypminor->yy414)); +destroyCreateTableSql((yypminor->yy436)); } break; - case 249: /* selcollist */ - case 261: /* sclp */ - case 271: /* exprlist */ + case 250: /* select */ { -tSQLExprListDestroy((yypminor->yy290)); +doDestroyQuerySql((yypminor->yy4)); } break; - case 251: /* where_opt */ - case 257: /* having_opt */ - case 262: /* expr */ - case 272: /* expritem */ + case 253: /* selcollist */ + case 265: /* sclp */ + case 275: /* exprlist */ { -tSQLExprDestroy((yypminor->yy64)); +tSqlExprListDestroy((yypminor->yy382)); } break; - case 260: /* union */ + case 255: /* where_opt */ + case 261: /* having_opt */ + case 266: /* expr */ + case 276: /* expritem */ { -destroyAllSelectClause((yypminor->yy231)); +tSqlExprDestroy((yypminor->yy522)); } break; - case 267: /* sortitem */ + case 264: /* union */ { -tVariantDestroy(&(yypminor->yy134)); +destroyAllSelectClause((yypminor->yy13)); +} + break; + case 271: /* sortitem */ +{ +tVariantDestroy(&(yypminor->yy488)); } break; /********* End destructor definitions *****************************************/ @@ -1698,239 +1719,245 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 208, -1 }, /* (0) program ::= cmd */ - { 209, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 209, -2 }, /* (2) cmd ::= SHOW MNODES */ - { 209, -2 }, /* (3) cmd ::= SHOW DNODES */ - { 209, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ - { 209, -2 }, /* (5) cmd ::= SHOW USERS */ - { 209, -2 }, /* (6) cmd ::= SHOW MODULES */ - { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ - { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ - { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ - { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ - { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ - { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ - { 209, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - { 210, 0 }, /* (15) dbPrefix ::= */ - { 210, -2 }, /* (16) dbPrefix ::= ids DOT */ - { 212, 0 }, /* (17) cpxName ::= */ - { 212, -2 }, /* (18) cpxName ::= DOT ids */ - { 209, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ - { 209, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ - { 209, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ - { 209, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 209, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ - { 209, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 209, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ - { 209, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 209, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ - { 209, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ - { 209, -3 }, /* (29) cmd ::= DROP DNODE ids */ - { 209, -3 }, /* (30) cmd ::= DROP USER ids */ - { 209, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ - { 209, -2 }, /* (32) cmd ::= USE ids */ - { 209, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ - { 209, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ - { 209, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 209, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ - { 209, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ - { 209, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ - { 209, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ - { 209, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 209, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 209, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 211, -1 }, /* (43) ids ::= ID */ - { 211, -1 }, /* (44) ids ::= STRING */ - { 213, -2 }, /* (45) ifexists ::= IF EXISTS */ - { 213, 0 }, /* (46) ifexists ::= */ - { 216, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ - { 216, 0 }, /* (48) ifnotexists ::= */ - { 209, -3 }, /* (49) cmd ::= CREATE DNODE ids */ - { 209, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 209, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 209, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ - { 218, 0 }, /* (53) pps ::= */ - { 218, -2 }, /* (54) pps ::= PPS INTEGER */ - { 219, 0 }, /* (55) tseries ::= */ - { 219, -2 }, /* (56) tseries ::= TSERIES INTEGER */ - { 220, 0 }, /* (57) dbs ::= */ - { 220, -2 }, /* (58) dbs ::= DBS INTEGER */ - { 221, 0 }, /* (59) streams ::= */ - { 221, -2 }, /* (60) streams ::= STREAMS INTEGER */ - { 222, 0 }, /* (61) storage ::= */ - { 222, -2 }, /* (62) storage ::= STORAGE INTEGER */ - { 223, 0 }, /* (63) qtime ::= */ - { 223, -2 }, /* (64) qtime ::= QTIME INTEGER */ - { 224, 0 }, /* (65) users ::= */ - { 224, -2 }, /* (66) users ::= USERS INTEGER */ - { 225, 0 }, /* (67) conns ::= */ - { 225, -2 }, /* (68) conns ::= CONNS INTEGER */ - { 226, 0 }, /* (69) state ::= */ - { 226, -2 }, /* (70) state ::= STATE ids */ - { 215, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 227, -2 }, /* (72) keep ::= KEEP tagitemlist */ - { 229, -2 }, /* (73) cache ::= CACHE INTEGER */ - { 230, -2 }, /* (74) replica ::= REPLICA INTEGER */ - { 231, -2 }, /* (75) quorum ::= QUORUM INTEGER */ - { 232, -2 }, /* (76) days ::= DAYS INTEGER */ - { 233, -2 }, /* (77) minrows ::= MINROWS INTEGER */ - { 234, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ - { 235, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ - { 236, -2 }, /* (80) ctime ::= CTIME INTEGER */ - { 237, -2 }, /* (81) wal ::= WAL INTEGER */ - { 238, -2 }, /* (82) fsync ::= FSYNC INTEGER */ - { 239, -2 }, /* (83) comp ::= COMP INTEGER */ - { 240, -2 }, /* (84) prec ::= PRECISION STRING */ - { 241, -2 }, /* (85) update ::= UPDATE INTEGER */ - { 217, 0 }, /* (86) db_optr ::= */ - { 217, -2 }, /* (87) db_optr ::= db_optr cache */ - { 217, -2 }, /* (88) db_optr ::= db_optr replica */ - { 217, -2 }, /* (89) db_optr ::= db_optr quorum */ - { 217, -2 }, /* (90) db_optr ::= db_optr days */ - { 217, -2 }, /* (91) db_optr ::= db_optr minrows */ - { 217, -2 }, /* (92) db_optr ::= db_optr maxrows */ - { 217, -2 }, /* (93) db_optr ::= db_optr blocks */ - { 217, -2 }, /* (94) db_optr ::= db_optr ctime */ - { 217, -2 }, /* (95) db_optr ::= db_optr wal */ - { 217, -2 }, /* (96) db_optr ::= db_optr fsync */ - { 217, -2 }, /* (97) db_optr ::= db_optr comp */ - { 217, -2 }, /* (98) db_optr ::= db_optr prec */ - { 217, -2 }, /* (99) db_optr ::= db_optr keep */ - { 217, -2 }, /* (100) db_optr ::= db_optr update */ - { 214, 0 }, /* (101) alter_db_optr ::= */ - { 214, -2 }, /* (102) alter_db_optr ::= alter_db_optr replica */ - { 214, -2 }, /* (103) alter_db_optr ::= alter_db_optr quorum */ - { 214, -2 }, /* (104) alter_db_optr ::= alter_db_optr keep */ - { 214, -2 }, /* (105) alter_db_optr ::= alter_db_optr blocks */ - { 214, -2 }, /* (106) alter_db_optr ::= alter_db_optr comp */ - { 214, -2 }, /* (107) alter_db_optr ::= alter_db_optr wal */ - { 214, -2 }, /* (108) alter_db_optr ::= alter_db_optr fsync */ - { 214, -2 }, /* (109) alter_db_optr ::= alter_db_optr update */ - { 242, -1 }, /* (110) typename ::= ids */ - { 242, -4 }, /* (111) typename ::= ids LP signed RP */ - { 243, -1 }, /* (112) signed ::= INTEGER */ - { 243, -2 }, /* (113) signed ::= PLUS INTEGER */ - { 243, -2 }, /* (114) signed ::= MINUS INTEGER */ - { 209, -6 }, /* (115) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - { 244, -3 }, /* (116) create_table_args ::= LP columnlist RP */ - { 244, -7 }, /* (117) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - { 244, -7 }, /* (118) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - { 244, -2 }, /* (119) create_table_args ::= AS select */ - { 245, -3 }, /* (120) columnlist ::= columnlist COMMA column */ - { 245, -1 }, /* (121) columnlist ::= column */ - { 247, -2 }, /* (122) column ::= ids typename */ - { 228, -3 }, /* (123) tagitemlist ::= tagitemlist COMMA tagitem */ - { 228, -1 }, /* (124) tagitemlist ::= tagitem */ - { 248, -1 }, /* (125) tagitem ::= INTEGER */ - { 248, -1 }, /* (126) tagitem ::= FLOAT */ - { 248, -1 }, /* (127) tagitem ::= STRING */ - { 248, -1 }, /* (128) tagitem ::= BOOL */ - { 248, -1 }, /* (129) tagitem ::= NULL */ - { 248, -2 }, /* (130) tagitem ::= MINUS INTEGER */ - { 248, -2 }, /* (131) tagitem ::= MINUS FLOAT */ - { 248, -2 }, /* (132) tagitem ::= PLUS INTEGER */ - { 248, -2 }, /* (133) tagitem ::= PLUS FLOAT */ - { 246, -12 }, /* (134) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - { 260, -1 }, /* (135) union ::= select */ - { 260, -3 }, /* (136) union ::= LP union RP */ - { 260, -4 }, /* (137) union ::= union UNION ALL select */ - { 260, -6 }, /* (138) union ::= union UNION ALL LP select RP */ - { 209, -1 }, /* (139) cmd ::= union */ - { 246, -2 }, /* (140) select ::= SELECT selcollist */ - { 261, -2 }, /* (141) sclp ::= selcollist COMMA */ - { 261, 0 }, /* (142) sclp ::= */ - { 249, -3 }, /* (143) selcollist ::= sclp expr as */ - { 249, -2 }, /* (144) selcollist ::= sclp STAR */ - { 263, -2 }, /* (145) as ::= AS ids */ - { 263, -1 }, /* (146) as ::= ids */ - { 263, 0 }, /* (147) as ::= */ - { 250, -2 }, /* (148) from ::= FROM tablelist */ - { 264, -2 }, /* (149) tablelist ::= ids cpxName */ - { 264, -3 }, /* (150) tablelist ::= ids cpxName ids */ - { 264, -4 }, /* (151) tablelist ::= tablelist COMMA ids cpxName */ - { 264, -5 }, /* (152) tablelist ::= tablelist COMMA ids cpxName ids */ - { 265, -1 }, /* (153) tmvar ::= VARIABLE */ - { 252, -4 }, /* (154) interval_opt ::= INTERVAL LP tmvar RP */ - { 252, -6 }, /* (155) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - { 252, 0 }, /* (156) interval_opt ::= */ - { 253, 0 }, /* (157) fill_opt ::= */ - { 253, -6 }, /* (158) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 253, -4 }, /* (159) fill_opt ::= FILL LP ID RP */ - { 254, -4 }, /* (160) sliding_opt ::= SLIDING LP tmvar RP */ - { 254, 0 }, /* (161) sliding_opt ::= */ - { 256, 0 }, /* (162) orderby_opt ::= */ - { 256, -3 }, /* (163) orderby_opt ::= ORDER BY sortlist */ - { 266, -4 }, /* (164) sortlist ::= sortlist COMMA item sortorder */ - { 266, -2 }, /* (165) sortlist ::= item sortorder */ - { 268, -2 }, /* (166) item ::= ids cpxName */ - { 269, -1 }, /* (167) sortorder ::= ASC */ - { 269, -1 }, /* (168) sortorder ::= DESC */ - { 269, 0 }, /* (169) sortorder ::= */ - { 255, 0 }, /* (170) groupby_opt ::= */ - { 255, -3 }, /* (171) groupby_opt ::= GROUP BY grouplist */ - { 270, -3 }, /* (172) grouplist ::= grouplist COMMA item */ - { 270, -1 }, /* (173) grouplist ::= item */ - { 257, 0 }, /* (174) having_opt ::= */ - { 257, -2 }, /* (175) having_opt ::= HAVING expr */ - { 259, 0 }, /* (176) limit_opt ::= */ - { 259, -2 }, /* (177) limit_opt ::= LIMIT signed */ - { 259, -4 }, /* (178) limit_opt ::= LIMIT signed OFFSET signed */ - { 259, -4 }, /* (179) limit_opt ::= LIMIT signed COMMA signed */ - { 258, 0 }, /* (180) slimit_opt ::= */ - { 258, -2 }, /* (181) slimit_opt ::= SLIMIT signed */ - { 258, -4 }, /* (182) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 258, -4 }, /* (183) slimit_opt ::= SLIMIT signed COMMA signed */ - { 251, 0 }, /* (184) where_opt ::= */ - { 251, -2 }, /* (185) where_opt ::= WHERE expr */ - { 262, -3 }, /* (186) expr ::= LP expr RP */ - { 262, -1 }, /* (187) expr ::= ID */ - { 262, -3 }, /* (188) expr ::= ID DOT ID */ - { 262, -3 }, /* (189) expr ::= ID DOT STAR */ - { 262, -1 }, /* (190) expr ::= INTEGER */ - { 262, -2 }, /* (191) expr ::= MINUS INTEGER */ - { 262, -2 }, /* (192) expr ::= PLUS INTEGER */ - { 262, -1 }, /* (193) expr ::= FLOAT */ - { 262, -2 }, /* (194) expr ::= MINUS FLOAT */ - { 262, -2 }, /* (195) expr ::= PLUS FLOAT */ - { 262, -1 }, /* (196) expr ::= STRING */ - { 262, -1 }, /* (197) expr ::= NOW */ - { 262, -1 }, /* (198) expr ::= VARIABLE */ - { 262, -1 }, /* (199) expr ::= BOOL */ - { 262, -4 }, /* (200) expr ::= ID LP exprlist RP */ - { 262, -4 }, /* (201) expr ::= ID LP STAR RP */ - { 262, -3 }, /* (202) expr ::= expr IS NULL */ - { 262, -4 }, /* (203) expr ::= expr IS NOT NULL */ - { 262, -3 }, /* (204) expr ::= expr LT expr */ - { 262, -3 }, /* (205) expr ::= expr GT expr */ - { 262, -3 }, /* (206) expr ::= expr LE expr */ - { 262, -3 }, /* (207) expr ::= expr GE expr */ - { 262, -3 }, /* (208) expr ::= expr NE expr */ - { 262, -3 }, /* (209) expr ::= expr EQ expr */ - { 262, -3 }, /* (210) expr ::= expr AND expr */ - { 262, -3 }, /* (211) expr ::= expr OR expr */ - { 262, -3 }, /* (212) expr ::= expr PLUS expr */ - { 262, -3 }, /* (213) expr ::= expr MINUS expr */ - { 262, -3 }, /* (214) expr ::= expr STAR expr */ - { 262, -3 }, /* (215) expr ::= expr SLASH expr */ - { 262, -3 }, /* (216) expr ::= expr REM expr */ - { 262, -3 }, /* (217) expr ::= expr LIKE expr */ - { 262, -5 }, /* (218) expr ::= expr IN LP exprlist RP */ - { 271, -3 }, /* (219) exprlist ::= exprlist COMMA expritem */ - { 271, -1 }, /* (220) exprlist ::= expritem */ - { 272, -1 }, /* (221) expritem ::= expr */ - { 272, 0 }, /* (222) expritem ::= */ - { 209, -3 }, /* (223) cmd ::= RESET QUERY CACHE */ - { 209, -7 }, /* (224) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 209, -7 }, /* (225) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 209, -7 }, /* (226) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 209, -7 }, /* (227) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 209, -8 }, /* (228) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 209, -9 }, /* (229) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 209, -3 }, /* (230) cmd ::= KILL CONNECTION INTEGER */ - { 209, -5 }, /* (231) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 209, -5 }, /* (232) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 209, -1 }, /* (0) program ::= cmd */ + { 210, -2 }, /* (1) cmd ::= SHOW DATABASES */ + { 210, -2 }, /* (2) cmd ::= SHOW MNODES */ + { 210, -2 }, /* (3) cmd ::= SHOW DNODES */ + { 210, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ + { 210, -2 }, /* (5) cmd ::= SHOW USERS */ + { 210, -2 }, /* (6) cmd ::= SHOW MODULES */ + { 210, -2 }, /* (7) cmd ::= SHOW QUERIES */ + { 210, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ + { 210, -2 }, /* (9) cmd ::= SHOW STREAMS */ + { 210, -2 }, /* (10) cmd ::= SHOW VARIABLES */ + { 210, -2 }, /* (11) cmd ::= SHOW SCORES */ + { 210, -2 }, /* (12) cmd ::= SHOW GRANTS */ + { 210, -2 }, /* (13) cmd ::= SHOW VNODES */ + { 210, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ + { 211, 0 }, /* (15) dbPrefix ::= */ + { 211, -2 }, /* (16) dbPrefix ::= ids DOT */ + { 213, 0 }, /* (17) cpxName ::= */ + { 213, -2 }, /* (18) cpxName ::= DOT ids */ + { 210, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ + { 210, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ + { 210, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ + { 210, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + { 210, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ + { 210, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + { 210, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ + { 210, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ + { 210, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ + { 210, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ + { 210, -3 }, /* (29) cmd ::= DROP DNODE ids */ + { 210, -3 }, /* (30) cmd ::= DROP USER ids */ + { 210, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ + { 210, -2 }, /* (32) cmd ::= USE ids */ + { 210, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ + { 210, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ + { 210, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ + { 210, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ + { 210, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ + { 210, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ + { 210, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ + { 210, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ + { 210, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ + { 210, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + { 212, -1 }, /* (43) ids ::= ID */ + { 212, -1 }, /* (44) ids ::= STRING */ + { 214, -2 }, /* (45) ifexists ::= IF EXISTS */ + { 214, 0 }, /* (46) ifexists ::= */ + { 217, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ + { 217, 0 }, /* (48) ifnotexists ::= */ + { 210, -3 }, /* (49) cmd ::= CREATE DNODE ids */ + { 210, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + { 210, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + { 210, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ + { 219, 0 }, /* (53) pps ::= */ + { 219, -2 }, /* (54) pps ::= PPS INTEGER */ + { 220, 0 }, /* (55) tseries ::= */ + { 220, -2 }, /* (56) tseries ::= TSERIES INTEGER */ + { 221, 0 }, /* (57) dbs ::= */ + { 221, -2 }, /* (58) dbs ::= DBS INTEGER */ + { 222, 0 }, /* (59) streams ::= */ + { 222, -2 }, /* (60) streams ::= STREAMS INTEGER */ + { 223, 0 }, /* (61) storage ::= */ + { 223, -2 }, /* (62) storage ::= STORAGE INTEGER */ + { 224, 0 }, /* (63) qtime ::= */ + { 224, -2 }, /* (64) qtime ::= QTIME INTEGER */ + { 225, 0 }, /* (65) users ::= */ + { 225, -2 }, /* (66) users ::= USERS INTEGER */ + { 226, 0 }, /* (67) conns ::= */ + { 226, -2 }, /* (68) conns ::= CONNS INTEGER */ + { 227, 0 }, /* (69) state ::= */ + { 227, -2 }, /* (70) state ::= STATE ids */ + { 216, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + { 228, -2 }, /* (72) keep ::= KEEP tagitemlist */ + { 230, -2 }, /* (73) cache ::= CACHE INTEGER */ + { 231, -2 }, /* (74) replica ::= REPLICA INTEGER */ + { 232, -2 }, /* (75) quorum ::= QUORUM INTEGER */ + { 233, -2 }, /* (76) days ::= DAYS INTEGER */ + { 234, -2 }, /* (77) minrows ::= MINROWS INTEGER */ + { 235, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ + { 236, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ + { 237, -2 }, /* (80) ctime ::= CTIME INTEGER */ + { 238, -2 }, /* (81) wal ::= WAL INTEGER */ + { 239, -2 }, /* (82) fsync ::= FSYNC INTEGER */ + { 240, -2 }, /* (83) comp ::= COMP INTEGER */ + { 241, -2 }, /* (84) prec ::= PRECISION STRING */ + { 242, -2 }, /* (85) update ::= UPDATE INTEGER */ + { 243, -2 }, /* (86) cachelast ::= CACHELAST INTEGER */ + { 218, 0 }, /* (87) db_optr ::= */ + { 218, -2 }, /* (88) db_optr ::= db_optr cache */ + { 218, -2 }, /* (89) db_optr ::= db_optr replica */ + { 218, -2 }, /* (90) db_optr ::= db_optr quorum */ + { 218, -2 }, /* (91) db_optr ::= db_optr days */ + { 218, -2 }, /* (92) db_optr ::= db_optr minrows */ + { 218, -2 }, /* (93) db_optr ::= db_optr maxrows */ + { 218, -2 }, /* (94) db_optr ::= db_optr blocks */ + { 218, -2 }, /* (95) db_optr ::= db_optr ctime */ + { 218, -2 }, /* (96) db_optr ::= db_optr wal */ + { 218, -2 }, /* (97) db_optr ::= db_optr fsync */ + { 218, -2 }, /* (98) db_optr ::= db_optr comp */ + { 218, -2 }, /* (99) db_optr ::= db_optr prec */ + { 218, -2 }, /* (100) db_optr ::= db_optr keep */ + { 218, -2 }, /* (101) db_optr ::= db_optr update */ + { 218, -2 }, /* (102) db_optr ::= db_optr cachelast */ + { 215, 0 }, /* (103) alter_db_optr ::= */ + { 215, -2 }, /* (104) alter_db_optr ::= alter_db_optr replica */ + { 215, -2 }, /* (105) alter_db_optr ::= alter_db_optr quorum */ + { 215, -2 }, /* (106) alter_db_optr ::= alter_db_optr keep */ + { 215, -2 }, /* (107) alter_db_optr ::= alter_db_optr blocks */ + { 215, -2 }, /* (108) alter_db_optr ::= alter_db_optr comp */ + { 215, -2 }, /* (109) alter_db_optr ::= alter_db_optr wal */ + { 215, -2 }, /* (110) alter_db_optr ::= alter_db_optr fsync */ + { 215, -2 }, /* (111) alter_db_optr ::= alter_db_optr update */ + { 215, -2 }, /* (112) alter_db_optr ::= alter_db_optr cachelast */ + { 244, -1 }, /* (113) typename ::= ids */ + { 244, -4 }, /* (114) typename ::= ids LP signed RP */ + { 245, -1 }, /* (115) signed ::= INTEGER */ + { 245, -2 }, /* (116) signed ::= PLUS INTEGER */ + { 245, -2 }, /* (117) signed ::= MINUS INTEGER */ + { 210, -3 }, /* (118) cmd ::= CREATE TABLE create_table_args */ + { 210, -3 }, /* (119) cmd ::= CREATE TABLE create_table_list */ + { 247, -1 }, /* (120) create_table_list ::= create_from_stable */ + { 247, -2 }, /* (121) create_table_list ::= create_table_list create_from_stable */ + { 246, -6 }, /* (122) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + { 246, -10 }, /* (123) create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + { 248, -10 }, /* (124) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + { 246, -5 }, /* (125) create_table_args ::= ifnotexists ids cpxName AS select */ + { 249, -3 }, /* (126) columnlist ::= columnlist COMMA column */ + { 249, -1 }, /* (127) columnlist ::= column */ + { 251, -2 }, /* (128) column ::= ids typename */ + { 229, -3 }, /* (129) tagitemlist ::= tagitemlist COMMA tagitem */ + { 229, -1 }, /* (130) tagitemlist ::= tagitem */ + { 252, -1 }, /* (131) tagitem ::= INTEGER */ + { 252, -1 }, /* (132) tagitem ::= FLOAT */ + { 252, -1 }, /* (133) tagitem ::= STRING */ + { 252, -1 }, /* (134) tagitem ::= BOOL */ + { 252, -1 }, /* (135) tagitem ::= NULL */ + { 252, -2 }, /* (136) tagitem ::= MINUS INTEGER */ + { 252, -2 }, /* (137) tagitem ::= MINUS FLOAT */ + { 252, -2 }, /* (138) tagitem ::= PLUS INTEGER */ + { 252, -2 }, /* (139) tagitem ::= PLUS FLOAT */ + { 250, -12 }, /* (140) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + { 264, -1 }, /* (141) union ::= select */ + { 264, -3 }, /* (142) union ::= LP union RP */ + { 264, -4 }, /* (143) union ::= union UNION ALL select */ + { 264, -6 }, /* (144) union ::= union UNION ALL LP select RP */ + { 210, -1 }, /* (145) cmd ::= union */ + { 250, -2 }, /* (146) select ::= SELECT selcollist */ + { 265, -2 }, /* (147) sclp ::= selcollist COMMA */ + { 265, 0 }, /* (148) sclp ::= */ + { 253, -3 }, /* (149) selcollist ::= sclp expr as */ + { 253, -2 }, /* (150) selcollist ::= sclp STAR */ + { 267, -2 }, /* (151) as ::= AS ids */ + { 267, -1 }, /* (152) as ::= ids */ + { 267, 0 }, /* (153) as ::= */ + { 254, -2 }, /* (154) from ::= FROM tablelist */ + { 268, -2 }, /* (155) tablelist ::= ids cpxName */ + { 268, -3 }, /* (156) tablelist ::= ids cpxName ids */ + { 268, -4 }, /* (157) tablelist ::= tablelist COMMA ids cpxName */ + { 268, -5 }, /* (158) tablelist ::= tablelist COMMA ids cpxName ids */ + { 269, -1 }, /* (159) tmvar ::= VARIABLE */ + { 256, -4 }, /* (160) interval_opt ::= INTERVAL LP tmvar RP */ + { 256, -6 }, /* (161) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + { 256, 0 }, /* (162) interval_opt ::= */ + { 257, 0 }, /* (163) fill_opt ::= */ + { 257, -6 }, /* (164) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 257, -4 }, /* (165) fill_opt ::= FILL LP ID RP */ + { 258, -4 }, /* (166) sliding_opt ::= SLIDING LP tmvar RP */ + { 258, 0 }, /* (167) sliding_opt ::= */ + { 260, 0 }, /* (168) orderby_opt ::= */ + { 260, -3 }, /* (169) orderby_opt ::= ORDER BY sortlist */ + { 270, -4 }, /* (170) sortlist ::= sortlist COMMA item sortorder */ + { 270, -2 }, /* (171) sortlist ::= item sortorder */ + { 272, -2 }, /* (172) item ::= ids cpxName */ + { 273, -1 }, /* (173) sortorder ::= ASC */ + { 273, -1 }, /* (174) sortorder ::= DESC */ + { 273, 0 }, /* (175) sortorder ::= */ + { 259, 0 }, /* (176) groupby_opt ::= */ + { 259, -3 }, /* (177) groupby_opt ::= GROUP BY grouplist */ + { 274, -3 }, /* (178) grouplist ::= grouplist COMMA item */ + { 274, -1 }, /* (179) grouplist ::= item */ + { 261, 0 }, /* (180) having_opt ::= */ + { 261, -2 }, /* (181) having_opt ::= HAVING expr */ + { 263, 0 }, /* (182) limit_opt ::= */ + { 263, -2 }, /* (183) limit_opt ::= LIMIT signed */ + { 263, -4 }, /* (184) limit_opt ::= LIMIT signed OFFSET signed */ + { 263, -4 }, /* (185) limit_opt ::= LIMIT signed COMMA signed */ + { 262, 0 }, /* (186) slimit_opt ::= */ + { 262, -2 }, /* (187) slimit_opt ::= SLIMIT signed */ + { 262, -4 }, /* (188) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 262, -4 }, /* (189) slimit_opt ::= SLIMIT signed COMMA signed */ + { 255, 0 }, /* (190) where_opt ::= */ + { 255, -2 }, /* (191) where_opt ::= WHERE expr */ + { 266, -3 }, /* (192) expr ::= LP expr RP */ + { 266, -1 }, /* (193) expr ::= ID */ + { 266, -3 }, /* (194) expr ::= ID DOT ID */ + { 266, -3 }, /* (195) expr ::= ID DOT STAR */ + { 266, -1 }, /* (196) expr ::= INTEGER */ + { 266, -2 }, /* (197) expr ::= MINUS INTEGER */ + { 266, -2 }, /* (198) expr ::= PLUS INTEGER */ + { 266, -1 }, /* (199) expr ::= FLOAT */ + { 266, -2 }, /* (200) expr ::= MINUS FLOAT */ + { 266, -2 }, /* (201) expr ::= PLUS FLOAT */ + { 266, -1 }, /* (202) expr ::= STRING */ + { 266, -1 }, /* (203) expr ::= NOW */ + { 266, -1 }, /* (204) expr ::= VARIABLE */ + { 266, -1 }, /* (205) expr ::= BOOL */ + { 266, -4 }, /* (206) expr ::= ID LP exprlist RP */ + { 266, -4 }, /* (207) expr ::= ID LP STAR RP */ + { 266, -3 }, /* (208) expr ::= expr IS NULL */ + { 266, -4 }, /* (209) expr ::= expr IS NOT NULL */ + { 266, -3 }, /* (210) expr ::= expr LT expr */ + { 266, -3 }, /* (211) expr ::= expr GT expr */ + { 266, -3 }, /* (212) expr ::= expr LE expr */ + { 266, -3 }, /* (213) expr ::= expr GE expr */ + { 266, -3 }, /* (214) expr ::= expr NE expr */ + { 266, -3 }, /* (215) expr ::= expr EQ expr */ + { 266, -3 }, /* (216) expr ::= expr AND expr */ + { 266, -3 }, /* (217) expr ::= expr OR expr */ + { 266, -3 }, /* (218) expr ::= expr PLUS expr */ + { 266, -3 }, /* (219) expr ::= expr MINUS expr */ + { 266, -3 }, /* (220) expr ::= expr STAR expr */ + { 266, -3 }, /* (221) expr ::= expr SLASH expr */ + { 266, -3 }, /* (222) expr ::= expr REM expr */ + { 266, -3 }, /* (223) expr ::= expr LIKE expr */ + { 266, -5 }, /* (224) expr ::= expr IN LP exprlist RP */ + { 275, -3 }, /* (225) exprlist ::= exprlist COMMA expritem */ + { 275, -1 }, /* (226) exprlist ::= expritem */ + { 276, -1 }, /* (227) expritem ::= expr */ + { 276, 0 }, /* (228) expritem ::= */ + { 210, -3 }, /* (229) cmd ::= RESET QUERY CACHE */ + { 210, -7 }, /* (230) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 210, -7 }, /* (231) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 210, -7 }, /* (232) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 210, -7 }, /* (233) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 210, -8 }, /* (234) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 210, -9 }, /* (235) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 210, -3 }, /* (236) cmd ::= KILL CONNECTION INTEGER */ + { 210, -5 }, /* (237) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 210, -5 }, /* (238) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2011,6 +2038,7 @@ static void yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* program ::= cmd */ + case 118: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==118); {} break; case 1: /* cmd ::= SHOW DATABASES */ @@ -2097,32 +2125,32 @@ static void yy_reduce( case 24: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SStrToken token; - setDBName(&token, &yymsp[-3].minor.yy0); + setDbName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; case 25: /* cmd ::= SHOW dbPrefix VGROUPS */ { SStrToken token; - setDBName(&token, &yymsp[-1].minor.yy0); + setDbName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; case 26: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { SStrToken token; - setDBName(&token, &yymsp[-2].minor.yy0); + setDbName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= DROP TABLE ifexists ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDropDBTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0); + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0); } break; case 28: /* cmd ::= DROP DATABASE ifexists ids */ -{ setDropDBTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0); } +{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0); } break; case 29: /* cmd ::= DROP DNODE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } @@ -2143,10 +2171,10 @@ static void yy_reduce( } break; case 34: /* cmd ::= ALTER USER ids PASS ids */ -{ setAlterUserSQL(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } +{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; case 35: /* cmd ::= ALTER USER ids PRIVILEGE ids */ -{ setAlterUserSQL(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} +{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} break; case 36: /* cmd ::= ALTER DNODE ids ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } @@ -2161,13 +2189,13 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 40: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy478, &t);} break; case 41: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy463);} break; case 42: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy463);} break; case 43: /* ids ::= ID */ case 44: /* ids ::= STRING */ yytestcase(yyruleno==44); @@ -2175,26 +2203,26 @@ static void yy_reduce( yymsp[0].minor.yy0 = yylhsminor.yy0; break; case 45: /* ifexists ::= IF EXISTS */ -{yymsp[-1].minor.yy0.n = 1;} +{ yymsp[-1].minor.yy0.n = 1;} break; case 46: /* ifexists ::= */ case 48: /* ifnotexists ::= */ yytestcase(yyruleno==48); -{yymsp[1].minor.yy0.n = 0;} +{ yymsp[1].minor.yy0.n = 0;} break; case 47: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy0.n = 1;} +{ yymsp[-2].minor.yy0.n = 1;} break; case 49: /* cmd ::= CREATE DNODE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 50: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} +{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy463);} break; case 51: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);} +{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy478, &yymsp[-2].minor.yy0);} break; case 52: /* cmd ::= CREATE USER ids PASS ids */ -{ setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} +{ setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} break; case 53: /* pps ::= */ case 55: /* tseries ::= */ yytestcase(yyruleno==55); @@ -2205,7 +2233,7 @@ static void yy_reduce( case 65: /* users ::= */ yytestcase(yyruleno==65); case 67: /* conns ::= */ yytestcase(yyruleno==67); case 69: /* state ::= */ yytestcase(yyruleno==69); -{yymsp[1].minor.yy0.n = 0; } +{ yymsp[1].minor.yy0.n = 0; } break; case 54: /* pps ::= PPS INTEGER */ case 56: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==56); @@ -2216,24 +2244,24 @@ static void yy_reduce( case 66: /* users ::= USERS INTEGER */ yytestcase(yyruleno==66); case 68: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==68); case 70: /* state ::= STATE ids */ yytestcase(yyruleno==70); -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; case 71: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy149.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy149.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy149.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy149.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy149.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy149.stat = yymsp[0].minor.yy0; -} - yymsp[-8].minor.yy149 = yylhsminor.yy149; + yylhsminor.yy463.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy463.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy463.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy463.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy463.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy463.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy463.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy463.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy463.stat = yymsp[0].minor.yy0; +} + yymsp[-8].minor.yy463 = yylhsminor.yy463; break; case 72: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy165 = yymsp[0].minor.yy165; } +{ yymsp[-1].minor.yy403 = yymsp[0].minor.yy403; } break; case 73: /* cache ::= CACHE INTEGER */ case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74); @@ -2248,579 +2276,617 @@ static void yy_reduce( case 83: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==83); case 84: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==84); case 85: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==85); + case 86: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==86); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 86: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy268);} - break; - case 87: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 88: /* db_optr ::= db_optr replica */ - case 102: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==102); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 89: /* db_optr ::= db_optr quorum */ - case 103: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==103); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 90: /* db_optr ::= db_optr days */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 91: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 92: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 93: /* db_optr ::= db_optr blocks */ - case 105: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==105); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 94: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 95: /* db_optr ::= db_optr wal */ - case 107: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==107); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 96: /* db_optr ::= db_optr fsync */ - case 108: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==108); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 97: /* db_optr ::= db_optr comp */ - case 106: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==106); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 98: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 99: /* db_optr ::= db_optr keep */ - case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy165; } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 100: /* db_optr ::= db_optr update */ - case 109: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==109); -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; - case 101: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy268);} - break; - case 110: /* typename ::= ids */ + case 87: /* db_optr ::= */ +{setDefaultCreateDbOption(&yymsp[1].minor.yy478);} + break; + case 88: /* db_optr ::= db_optr cache */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 89: /* db_optr ::= db_optr replica */ + case 104: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==104); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 90: /* db_optr ::= db_optr quorum */ + case 105: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==105); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 91: /* db_optr ::= db_optr days */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 92: /* db_optr ::= db_optr minrows */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 93: /* db_optr ::= db_optr maxrows */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 94: /* db_optr ::= db_optr blocks */ + case 107: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==107); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 95: /* db_optr ::= db_optr ctime */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 96: /* db_optr ::= db_optr wal */ + case 109: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==109); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 97: /* db_optr ::= db_optr fsync */ + case 110: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==110); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 98: /* db_optr ::= db_optr comp */ + case 108: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==108); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 99: /* db_optr ::= db_optr prec */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 100: /* db_optr ::= db_optr keep */ + case 106: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==106); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.keep = yymsp[0].minor.yy403; } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 101: /* db_optr ::= db_optr update */ + case 111: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==111); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 102: /* db_optr ::= db_optr cachelast */ + case 112: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==112); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 103: /* alter_db_optr ::= */ +{ setDefaultCreateDbOption(&yymsp[1].minor.yy478);} + break; + case 113: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSQLSetColumnType (&yylhsminor.yy223, &yymsp[0].minor.yy0); + tSqlSetColumnType (&yylhsminor.yy363, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy223 = yylhsminor.yy223; + yymsp[0].minor.yy363 = yylhsminor.yy363; break; - case 111: /* typename ::= ids LP signed RP */ + case 114: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy207 <= 0) { - yymsp[-3].minor.yy0.type = 0; - tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); - } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207; // negative value of name length - tSQLSetColumnType(&yylhsminor.yy223, &yymsp[-3].minor.yy0); - } + if (yymsp[-1].minor.yy387 <= 0) { + yymsp[-3].minor.yy0.type = 0; + tSqlSetColumnType(&yylhsminor.yy363, &yymsp[-3].minor.yy0); + } else { + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy387; // negative value of name length + tSqlSetColumnType(&yylhsminor.yy363, &yymsp[-3].minor.yy0); + } } - yymsp[-3].minor.yy223 = yylhsminor.yy223; + yymsp[-3].minor.yy363 = yylhsminor.yy363; + break; + case 115: /* signed ::= INTEGER */ +{ yylhsminor.yy387 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy387 = yylhsminor.yy387; + break; + case 116: /* signed ::= PLUS INTEGER */ +{ yymsp[-1].minor.yy387 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 112: /* signed ::= INTEGER */ -{ yylhsminor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy207 = yylhsminor.yy207; + case 117: /* signed ::= MINUS INTEGER */ +{ yymsp[-1].minor.yy387 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 113: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 119: /* cmd ::= CREATE TABLE create_table_list */ +{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy436;} break; - case 114: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} + case 120: /* create_table_list ::= create_from_stable */ +{ + SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); + pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); + + taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy84); + pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; + yylhsminor.yy436 = pCreateTable; +} + yymsp[0].minor.yy436 = yylhsminor.yy436; break; - case 115: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + case 121: /* create_table_list ::= create_table_list create_from_stable */ { - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); + taosArrayPush(yymsp[-1].minor.yy436->childTableInfo, &yymsp[0].minor.yy84); + yylhsminor.yy436 = yymsp[-1].minor.yy436; } + yymsp[-1].minor.yy436 = yylhsminor.yy436; break; - case 116: /* create_table_args ::= LP columnlist RP */ + case 122: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { - yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy165, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy436 = tSetCreateSqlElems(yymsp[-1].minor.yy403, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, yylhsminor.yy436, NULL, TSDB_SQL_CREATE_TABLE); + + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); } + yymsp[-5].minor.yy436 = yylhsminor.yy436; break; - case 117: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + case 123: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { - yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy165, yymsp[-1].minor.yy165, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy436 = tSetCreateSqlElems(yymsp[-5].minor.yy403, yymsp[-1].minor.yy403, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, yylhsminor.yy436, NULL, TSDB_SQL_CREATE_TABLE); + + yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; + setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } + yymsp[-9].minor.yy436 = yylhsminor.yy436; break; - case 118: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + case 124: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ { - yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy165, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; + yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; + yylhsminor.yy84 = createNewChildTableInfo(&yymsp[-5].minor.yy0, yymsp[-1].minor.yy403, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } + yymsp[-9].minor.yy84 = yylhsminor.yy84; break; - case 119: /* create_table_args ::= AS select */ + case 125: /* create_table_args ::= ifnotexists ids cpxName AS select */ { - yymsp[-1].minor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, yymsp[-1].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy436 = tSetCreateSqlElems(NULL, NULL, yymsp[0].minor.yy4, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, yylhsminor.yy436, NULL, TSDB_SQL_CREATE_TABLE); + + yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; + setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); } + yymsp[-4].minor.yy436 = yylhsminor.yy436; break; - case 120: /* columnlist ::= columnlist COMMA column */ -{taosArrayPush(yymsp[-2].minor.yy165, &yymsp[0].minor.yy223); yylhsminor.yy165 = yymsp[-2].minor.yy165; } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + case 126: /* columnlist ::= columnlist COMMA column */ +{taosArrayPush(yymsp[-2].minor.yy403, &yymsp[0].minor.yy363); yylhsminor.yy403 = yymsp[-2].minor.yy403; } + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 121: /* columnlist ::= column */ -{yylhsminor.yy165 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy165, &yymsp[0].minor.yy223);} - yymsp[0].minor.yy165 = yylhsminor.yy165; + case 127: /* columnlist ::= column */ +{yylhsminor.yy403 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy403, &yymsp[0].minor.yy363);} + yymsp[0].minor.yy403 = yylhsminor.yy403; break; - case 122: /* column ::= ids typename */ + case 128: /* column ::= ids typename */ { - tSQLSetColumnInfo(&yylhsminor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223); + tSqlSetColumnInfo(&yylhsminor.yy363, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy363); } - yymsp[-1].minor.yy223 = yylhsminor.yy223; + yymsp[-1].minor.yy363 = yylhsminor.yy363; break; - case 123: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1); } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + case 129: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yylhsminor.yy403 = tVariantListAppend(yymsp[-2].minor.yy403, &yymsp[0].minor.yy488, -1); } + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 124: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } - yymsp[0].minor.yy165 = yylhsminor.yy165; + case 130: /* tagitemlist ::= tagitem */ +{ yylhsminor.yy403 = tVariantListAppend(NULL, &yymsp[0].minor.yy488, -1); } + yymsp[0].minor.yy403 = yylhsminor.yy403; break; - case 125: /* tagitem ::= INTEGER */ - case 126: /* tagitem ::= FLOAT */ yytestcase(yyruleno==126); - case 127: /* tagitem ::= STRING */ yytestcase(yyruleno==127); - case 128: /* tagitem ::= BOOL */ yytestcase(yyruleno==128); -{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy134 = yylhsminor.yy134; + case 131: /* tagitem ::= INTEGER */ + case 132: /* tagitem ::= FLOAT */ yytestcase(yyruleno==132); + case 133: /* tagitem ::= STRING */ yytestcase(yyruleno==133); + case 134: /* tagitem ::= BOOL */ yytestcase(yyruleno==134); +{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy488, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy488 = yylhsminor.yy488; break; - case 129: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy134, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy134 = yylhsminor.yy134; + case 135: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy488, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy488 = yylhsminor.yy488; break; - case 130: /* tagitem ::= MINUS INTEGER */ - case 131: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==131); - case 132: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==132); - case 133: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==133); + case 136: /* tagitem ::= MINUS INTEGER */ + case 137: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==137); + case 138: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==138); + case 139: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==139); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy488, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy134 = yylhsminor.yy134; + yymsp[-1].minor.yy488 = yylhsminor.yy488; break; - case 134: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 140: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy165, yymsp[-8].minor.yy64, yymsp[-4].minor.yy165, yymsp[-3].minor.yy165, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy165, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216); + yylhsminor.yy4 = tSetQuerySqlElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy382, yymsp[-9].minor.yy403, yymsp[-8].minor.yy522, yymsp[-4].minor.yy403, yymsp[-3].minor.yy403, &yymsp[-7].minor.yy222, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy403, &yymsp[0].minor.yy404, &yymsp[-1].minor.yy404); } - yymsp[-11].minor.yy414 = yylhsminor.yy414; + yymsp[-11].minor.yy4 = yylhsminor.yy4; break; - case 135: /* union ::= select */ -{ yylhsminor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); } - yymsp[0].minor.yy231 = yylhsminor.yy231; + case 141: /* union ::= select */ +{ yylhsminor.yy13 = setSubclause(NULL, yymsp[0].minor.yy4); } + yymsp[0].minor.yy13 = yylhsminor.yy13; break; - case 136: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy231 = yymsp[-1].minor.yy231; } + case 142: /* union ::= LP union RP */ +{ yymsp[-2].minor.yy13 = yymsp[-1].minor.yy13; } break; - case 137: /* union ::= union UNION ALL select */ -{ yylhsminor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); } - yymsp[-3].minor.yy231 = yylhsminor.yy231; + case 143: /* union ::= union UNION ALL select */ +{ yylhsminor.yy13 = appendSelectClause(yymsp[-3].minor.yy13, yymsp[0].minor.yy4); } + yymsp[-3].minor.yy13 = yylhsminor.yy13; break; - case 138: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); } - yymsp[-5].minor.yy231 = yylhsminor.yy231; + case 144: /* union ::= union UNION ALL LP select RP */ +{ yylhsminor.yy13 = appendSelectClause(yymsp[-5].minor.yy13, yymsp[-1].minor.yy4); } + yymsp[-5].minor.yy13 = yylhsminor.yy13; break; - case 139: /* cmd ::= union */ -{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); } + case 145: /* cmd ::= union */ +{ setSqlInfo(pInfo, yymsp[0].minor.yy13, NULL, TSDB_SQL_SELECT); } break; - case 140: /* select ::= SELECT selcollist */ + case 146: /* select ::= SELECT selcollist */ { - yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy4 = tSetQuerySqlElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy382, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy414 = yylhsminor.yy414; + yymsp[-1].minor.yy4 = yylhsminor.yy4; break; - case 141: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy290 = yymsp[-1].minor.yy290;} - yymsp[-1].minor.yy290 = yylhsminor.yy290; + case 147: /* sclp ::= selcollist COMMA */ +{yylhsminor.yy382 = yymsp[-1].minor.yy382;} + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; - case 142: /* sclp ::= */ -{yymsp[1].minor.yy290 = 0;} + case 148: /* sclp ::= */ +{yymsp[1].minor.yy382 = 0;} break; - case 143: /* selcollist ::= sclp expr as */ + case 149: /* selcollist ::= sclp expr as */ { - yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy382 = tSqlExprListAppend(yymsp[-2].minor.yy382, yymsp[-1].minor.yy522, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy290 = yylhsminor.yy290; + yymsp[-2].minor.yy382 = yylhsminor.yy382; break; - case 144: /* selcollist ::= sclp STAR */ + case 150: /* selcollist ::= sclp STAR */ { - tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0); + tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); + yylhsminor.yy382 = tSqlExprListAppend(yymsp[-1].minor.yy382, pNode, 0); } - yymsp[-1].minor.yy290 = yylhsminor.yy290; + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; - case 145: /* as ::= AS ids */ + case 151: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 146: /* as ::= ids */ + case 152: /* as ::= ids */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 147: /* as ::= */ + case 153: /* as ::= */ { yymsp[1].minor.yy0.n = 0; } break; - case 148: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy165 = yymsp[0].minor.yy165;} + case 154: /* from ::= FROM tablelist */ +{yymsp[-1].minor.yy403 = yymsp[0].minor.yy403;} break; - case 149: /* tablelist ::= ids cpxName */ + case 155: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1); // table alias name + yylhsminor.yy403 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy165 = yylhsminor.yy165; + yymsp[-1].minor.yy403 = yylhsminor.yy403; break; - case 150: /* tablelist ::= ids cpxName ids */ + case 156: /* tablelist ::= ids cpxName ids */ { - toTSDBType(yymsp[-2].minor.yy0.type); - toTSDBType(yymsp[0].minor.yy0.type); - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1); + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy403 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 151: /* tablelist ::= tablelist COMMA ids cpxName */ + case 157: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yymsp[-3].minor.yy403, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy165 = yylhsminor.yy165; + yymsp[-3].minor.yy403 = yylhsminor.yy403; break; - case 152: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 158: /* tablelist ::= tablelist COMMA ids cpxName ids */ { - toTSDBType(yymsp[-2].minor.yy0.type); - toTSDBType(yymsp[0].minor.yy0.type); - yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy165 = tVariantListAppendToken(yymsp[-4].minor.yy165, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1); + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy403 = tVariantListAppendToken(yymsp[-4].minor.yy403, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy165 = yylhsminor.yy165; + yymsp[-4].minor.yy403 = yylhsminor.yy403; break; - case 153: /* tmvar ::= VARIABLE */ + case 159: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 154: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy532.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy532.offset.n = 0; yymsp[-3].minor.yy532.offset.z = NULL; yymsp[-3].minor.yy532.offset.type = 0;} + case 160: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yymsp[-3].minor.yy222.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy222.offset.n = 0; yymsp[-3].minor.yy222.offset.z = NULL; yymsp[-3].minor.yy222.offset.type = 0;} break; - case 155: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy532.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy532.offset = yymsp[-1].minor.yy0;} + case 161: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yymsp[-5].minor.yy222.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy222.offset = yymsp[-1].minor.yy0;} break; - case 156: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy532, 0, sizeof(yymsp[1].minor.yy532));} + case 162: /* interval_opt ::= */ +{memset(&yymsp[1].minor.yy222, 0, sizeof(yymsp[1].minor.yy222));} break; - case 157: /* fill_opt ::= */ -{yymsp[1].minor.yy165 = 0; } + case 163: /* fill_opt ::= */ +{yymsp[1].minor.yy403 = 0; } break; - case 158: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 164: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy165, &A, -1, 0); - yymsp[-5].minor.yy165 = yymsp[-1].minor.yy165; + tVariantListInsert(yymsp[-1].minor.yy403, &A, -1, 0); + yymsp[-5].minor.yy403 = yymsp[-1].minor.yy403; } break; - case 159: /* fill_opt ::= FILL LP ID RP */ + case 165: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy403 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 160: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 166: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 161: /* sliding_opt ::= */ + case 167: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 162: /* orderby_opt ::= */ - case 170: /* groupby_opt ::= */ yytestcase(yyruleno==170); -{yymsp[1].minor.yy165 = 0;} + case 168: /* orderby_opt ::= */ +{yymsp[1].minor.yy403 = 0;} break; - case 163: /* orderby_opt ::= ORDER BY sortlist */ - case 171: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==171); -{yymsp[-2].minor.yy165 = yymsp[0].minor.yy165;} + case 169: /* orderby_opt ::= ORDER BY sortlist */ +{yymsp[-2].minor.yy403 = yymsp[0].minor.yy403;} break; - case 164: /* sortlist ::= sortlist COMMA item sortorder */ + case 170: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy165 = tVariantListAppend(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); + yylhsminor.yy403 = tVariantListAppend(yymsp[-3].minor.yy403, &yymsp[-1].minor.yy488, yymsp[0].minor.yy70); } - yymsp[-3].minor.yy165 = yylhsminor.yy165; + yymsp[-3].minor.yy403 = yylhsminor.yy403; break; - case 165: /* sortlist ::= item sortorder */ + case 171: /* sortlist ::= item sortorder */ { - yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); + yylhsminor.yy403 = tVariantListAppend(NULL, &yymsp[-1].minor.yy488, yymsp[0].minor.yy70); } - yymsp[-1].minor.yy165 = yylhsminor.yy165; + yymsp[-1].minor.yy403 = yylhsminor.yy403; break; - case 166: /* item ::= ids cpxName */ + case 172: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy134, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy488, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy134 = yylhsminor.yy134; + yymsp[-1].minor.yy488 = yylhsminor.yy488; + break; + case 173: /* sortorder ::= ASC */ +{ yymsp[0].minor.yy70 = TSDB_ORDER_ASC; } + break; + case 174: /* sortorder ::= DESC */ +{ yymsp[0].minor.yy70 = TSDB_ORDER_DESC;} break; - case 167: /* sortorder ::= ASC */ -{yymsp[0].minor.yy46 = TSDB_ORDER_ASC; } + case 175: /* sortorder ::= */ +{ yymsp[1].minor.yy70 = TSDB_ORDER_ASC; } break; - case 168: /* sortorder ::= DESC */ -{yymsp[0].minor.yy46 = TSDB_ORDER_DESC;} + case 176: /* groupby_opt ::= */ +{ yymsp[1].minor.yy403 = 0;} break; - case 169: /* sortorder ::= */ -{yymsp[1].minor.yy46 = TSDB_ORDER_ASC;} + case 177: /* groupby_opt ::= GROUP BY grouplist */ +{ yymsp[-2].minor.yy403 = yymsp[0].minor.yy403;} break; - case 172: /* grouplist ::= grouplist COMMA item */ + case 178: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1); + yylhsminor.yy403 = tVariantListAppend(yymsp[-2].minor.yy403, &yymsp[0].minor.yy488, -1); } - yymsp[-2].minor.yy165 = yylhsminor.yy165; + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 173: /* grouplist ::= item */ + case 179: /* grouplist ::= item */ { - yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); + yylhsminor.yy403 = tVariantListAppend(NULL, &yymsp[0].minor.yy488, -1); } - yymsp[0].minor.yy165 = yylhsminor.yy165; + yymsp[0].minor.yy403 = yylhsminor.yy403; break; - case 174: /* having_opt ::= */ - case 184: /* where_opt ::= */ yytestcase(yyruleno==184); - case 222: /* expritem ::= */ yytestcase(yyruleno==222); -{yymsp[1].minor.yy64 = 0;} + case 180: /* having_opt ::= */ + case 190: /* where_opt ::= */ yytestcase(yyruleno==190); + case 228: /* expritem ::= */ yytestcase(yyruleno==228); +{yymsp[1].minor.yy522 = 0;} break; - case 175: /* having_opt ::= HAVING expr */ - case 185: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==185); -{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;} + case 181: /* having_opt ::= HAVING expr */ + case 191: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==191); +{yymsp[-1].minor.yy522 = yymsp[0].minor.yy522;} break; - case 176: /* limit_opt ::= */ - case 180: /* slimit_opt ::= */ yytestcase(yyruleno==180); -{yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;} + case 182: /* limit_opt ::= */ + case 186: /* slimit_opt ::= */ yytestcase(yyruleno==186); +{yymsp[1].minor.yy404.limit = -1; yymsp[1].minor.yy404.offset = 0;} break; - case 177: /* limit_opt ::= LIMIT signed */ - case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181); -{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} + case 183: /* limit_opt ::= LIMIT signed */ + case 187: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==187); +{yymsp[-1].minor.yy404.limit = yymsp[0].minor.yy387; yymsp[-1].minor.yy404.offset = 0;} break; - case 178: /* limit_opt ::= LIMIT signed OFFSET signed */ -{ yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} + case 184: /* limit_opt ::= LIMIT signed OFFSET signed */ +{ yymsp[-3].minor.yy404.limit = yymsp[-2].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[0].minor.yy387;} break; - case 179: /* limit_opt ::= LIMIT signed COMMA signed */ -{ yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} + case 185: /* limit_opt ::= LIMIT signed COMMA signed */ +{ yymsp[-3].minor.yy404.limit = yymsp[0].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[-2].minor.yy387;} break; - case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} + case 188: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ +{yymsp[-3].minor.yy404.limit = yymsp[-2].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[0].minor.yy387;} break; - case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */ -{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} + case 189: /* slimit_opt ::= SLIMIT signed COMMA signed */ +{yymsp[-3].minor.yy404.limit = yymsp[0].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[-2].minor.yy387;} break; - case 186: /* expr ::= LP expr RP */ -{yylhsminor.yy64 = yymsp[-1].minor.yy64; yylhsminor.yy64->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy64->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 192: /* expr ::= LP expr RP */ +{yylhsminor.yy522 = yymsp[-1].minor.yy522; yylhsminor.yy522->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy522->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 187: /* expr ::= ID */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 193: /* expr ::= ID */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 188: /* expr ::= ID DOT ID */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 194: /* expr ::= ID DOT ID */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 189: /* expr ::= ID DOT STAR */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 195: /* expr ::= ID DOT STAR */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 190: /* expr ::= INTEGER */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 196: /* expr ::= INTEGER */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 191: /* expr ::= MINUS INTEGER */ - case 192: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==192); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy64 = yylhsminor.yy64; + case 197: /* expr ::= MINUS INTEGER */ + case 198: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==198); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy522 = yylhsminor.yy522; break; - case 193: /* expr ::= FLOAT */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 199: /* expr ::= FLOAT */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 194: /* expr ::= MINUS FLOAT */ - case 195: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==195); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy64 = yylhsminor.yy64; + case 200: /* expr ::= MINUS FLOAT */ + case 201: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==201); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy522 = yylhsminor.yy522; break; - case 196: /* expr ::= STRING */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 202: /* expr ::= STRING */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 197: /* expr ::= NOW */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 203: /* expr ::= NOW */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 198: /* expr ::= VARIABLE */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 204: /* expr ::= VARIABLE */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 199: /* expr ::= BOOL */ -{yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 205: /* expr ::= BOOL */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 200: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy64 = yylhsminor.yy64; + case 206: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy522 = tSqlExprCreateFunction(yymsp[-1].minor.yy382, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy522 = yylhsminor.yy522; break; - case 201: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy64 = yylhsminor.yy64; + case 207: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy522 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy522 = yylhsminor.yy522; break; - case 202: /* expr ::= expr IS NULL */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, NULL, TK_ISNULL);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 208: /* expr ::= expr IS NULL */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, NULL, TK_ISNULL);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 203: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-3].minor.yy64, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy64 = yylhsminor.yy64; + case 209: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-3].minor.yy522, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy522 = yylhsminor.yy522; break; - case 204: /* expr ::= expr LT expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 210: /* expr ::= expr LT expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_LT);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 205: /* expr ::= expr GT expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 211: /* expr ::= expr GT expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_GT);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 206: /* expr ::= expr LE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 212: /* expr ::= expr LE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_LE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 207: /* expr ::= expr GE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 213: /* expr ::= expr GE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_GE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 208: /* expr ::= expr NE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 214: /* expr ::= expr NE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_NE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 209: /* expr ::= expr EQ expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 215: /* expr ::= expr EQ expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_EQ);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 210: /* expr ::= expr AND expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 216: /* expr ::= expr AND expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_AND);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 211: /* expr ::= expr OR expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 217: /* expr ::= expr OR expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_OR); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 212: /* expr ::= expr PLUS expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 218: /* expr ::= expr PLUS expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_PLUS); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 213: /* expr ::= expr MINUS expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 219: /* expr ::= expr MINUS expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_MINUS); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 214: /* expr ::= expr STAR expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 220: /* expr ::= expr STAR expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_STAR); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 215: /* expr ::= expr SLASH expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);} - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 221: /* expr ::= expr SLASH expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_DIVIDE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 216: /* expr ::= expr REM expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 222: /* expr ::= expr REM expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_REM); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 217: /* expr ::= expr LIKE expr */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); } - yymsp[-2].minor.yy64 = yylhsminor.yy64; + case 223: /* expr ::= expr LIKE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_LIKE); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 218: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); } - yymsp[-4].minor.yy64 = yylhsminor.yy64; + case 224: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-4].minor.yy522, (tSQLExpr*)yymsp[-1].minor.yy382, TK_IN); } + yymsp[-4].minor.yy522 = yylhsminor.yy522; break; - case 219: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);} - yymsp[-2].minor.yy290 = yylhsminor.yy290; + case 225: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy382 = tSqlExprListAppend(yymsp[-2].minor.yy382,yymsp[0].minor.yy522,0);} + yymsp[-2].minor.yy382 = yylhsminor.yy382; break; - case 220: /* exprlist ::= expritem */ -{yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);} - yymsp[0].minor.yy290 = yylhsminor.yy290; + case 226: /* exprlist ::= expritem */ +{yylhsminor.yy382 = tSqlExprListAppend(0,yymsp[0].minor.yy522,0);} + yymsp[0].minor.yy382 = yylhsminor.yy382; break; - case 221: /* expritem ::= expr */ -{yylhsminor.yy64 = yymsp[0].minor.yy64;} - yymsp[0].minor.yy64 = yylhsminor.yy64; + case 227: /* expritem ::= expr */ +{yylhsminor.yy522 = yymsp[0].minor.yy522;} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 223: /* cmd ::= RESET QUERY CACHE */ + case 229: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 224: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 230: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy403, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 225: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 231: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; toTSDBType(yymsp[0].minor.yy0.type); SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 226: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 232: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy403, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 227: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 233: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; toTSDBType(yymsp[0].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 228: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 234: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2830,30 +2896,30 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 229: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 235: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy488, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 230: /* cmd ::= KILL CONNECTION INTEGER */ -{setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} + case 236: /* cmd ::= KILL CONNECTION INTEGER */ +{setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 231: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} + case 237: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ +{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 232: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} + case 238: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ +{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: break; diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index 4a5f7fbbbed18b0a2c2cd7fff7ad3101cfa11c41..82fd4bcf9907eeb1286f9b6d0976e541e7b1a16b 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -21,19 +21,19 @@ TEST(testCase, histogram_binary_search) { pHisto->elems[i].val = i; } - int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1); assert(idx == 1); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9); assert(idx == 9); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20); assert(idx == 10); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1); assert(idx == 0); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); assert(idx == 4); free(pHisto); diff --git a/src/rpc/inc/rpcHead.h b/src/rpc/inc/rpcHead.h index 520edadc7dd072849720cad53c7f6f4ba605a06c..5b401ac54be194a38bad2f7680155f5edb8350b4 100644 --- a/src/rpc/inc/rpcHead.h +++ b/src/rpc/inc/rpcHead.h @@ -20,10 +20,6 @@ extern "C" { #endif -#define RPC_CONN_UDPS 0 -#define RPC_CONN_UDPC 1 -#define RPC_CONN_TCPS 2 -#define RPC_CONN_TCPC 3 #define RPC_CONN_TCP 2 extern int tsRpcOverhead; @@ -58,6 +54,7 @@ typedef struct { char empty[1]; // reserved uint8_t msgType; // message type int32_t msgLen; // message length including the header iteslf + uint32_t msgVer; int32_t code; // code in response message uint8_t content[0]; // message body starts from here } SRpcHead; diff --git a/src/rpc/inc/rpcLog.h b/src/rpc/inc/rpcLog.h index 10e847669146251ff5b3b4a5185bd09625f6ba07..6c4a281d2c0f0bf85975200c833acbac856dc40c 100644 --- a/src/rpc/inc/rpcLog.h +++ b/src/rpc/inc/rpcLog.h @@ -23,7 +23,7 @@ extern "C" { #include "tlog.h" extern int32_t rpcDebugFlag; -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; #define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 09d8f3bff1faad5596d85e931adea7f83670a48a..60a12c26b78626ed81cbc182d76c836c6ee74498 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -272,7 +272,7 @@ static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) } static void rpcLockCache(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { @@ -282,7 +282,7 @@ static void rpcLockCache(int64_t *lockedBy) { } static void rpcUnlockCache(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { assert(false); } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 587c079fe6201ac804aca35ab15d54bff968688e..13d6ed8ed5db31839bffb7b868ff22d9a65486b4 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -142,7 +142,6 @@ static int32_t tsRpcNum = 0; #define RPC_CONN_UDPC 1 #define RPC_CONN_TCPS 2 #define RPC_CONN_TCPC 3 -#define RPC_CONN_TCP 2 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { taosInitUdpConnection, @@ -313,7 +312,7 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } - tDebug("%s rpc is openned, threads:%d sessions:%d", pRpc->label, pRpc->numOfThreads, pInit->sessions); + tDebug("%s rpc is opened, threads:%d sessions:%d", pRpc->label, pRpc->numOfThreads, pInit->sessions); return pRpc; } @@ -404,7 +403,7 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64 if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META - || type == TSDB_MSG_TYPE_CM_SHOW ) + || type == TSDB_MSG_TYPE_CM_SHOW || type == TSDB_MSG_TYPE_DM_STATUS) pContext->connType = RPC_CONN_TCPC; pContext->rid = taosAddRef(tsRpcRefId, pContext); @@ -576,7 +575,7 @@ static void rpcFreeMsg(void *msg) { static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { SRpcConn *pConn; - uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); + uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -631,15 +630,19 @@ static void rpcReleaseConn(SRpcConn *pConn) { // if there is an outgoing message, free it if (pConn->outType && pConn->pReqMsg) { SRpcReqContext *pContext = pConn->pContext; - if (pContext->pRsp) { + if (pContext) { + if (pContext->pRsp) { // for synchronous API, post semaphore to unblock app - pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; - pContext->pRsp->pCont = NULL; - pContext->pRsp->contLen = 0; - tsem_post(pContext->pSem); + pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; + pContext->pRsp->pCont = NULL; + pContext->pRsp->contLen = 0; + tsem_post(pContext->pSem); + } + pContext->pConn = NULL; + taosRemoveRef(tsRpcRefId, pContext->rid); + } else { + assert(0); } - pContext->pConn = NULL; - taosRemoveRef(tsRpcRefId, pContext->rid); } } @@ -955,6 +958,11 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL; } + if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { + tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, taosMsg[pHead->msgType]); + terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL; + } + pConn = rpcGetConnObj(pRpc, sid, pRecv); if (pConn == NULL) { tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); @@ -1083,7 +1091,11 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { rpcCloseConn(pConn); } - tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); + if (pHead->msgType + 1 > 1 && pHead->msgType+1 < TSDB_MSG_TYPE_MAX) { + tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code); + } else { + tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], code); + } } } else { // msg is passed to app only parsing is ok rpcProcessIncomingMsg(pConn, pHead, pContext); @@ -1204,6 +1216,7 @@ static void rpcSendReqHead(SRpcConn *pConn) { pHead = (SRpcHead *)msg; pHead->version = 1; pHead->msgType = pConn->outType; + pHead->msgVer = htonl(tsVersion >> 8); pHead->spi = pConn->spi; pHead->encrypt = 0; pHead->tranId = pConn->outTranId; @@ -1274,6 +1287,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { // set the message header pHead->version = 1; + pHead->msgVer = htonl(tsVersion >> 8); pHead->msgType = msgType; pHead->encrypt = 0; pConn->tranId++; @@ -1590,7 +1604,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { } static void rpcLockConn(SRpcConn *pConn) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(&(pConn->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { @@ -1600,7 +1614,7 @@ static void rpcLockConn(SRpcConn *pConn) { } static void rpcUnlockConn(SRpcConn *pConn) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(pConn->lockedBy), tid, 0) != tid) { assert(false); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 2850046d050cba4adceeac615c0672d36dfed7d9..178b96c423641ccc6be84c57ca71fe7b85fd7a33 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -242,7 +242,14 @@ static void *taosAcceptTcpConnection(void *arg) { taosKeepTcpAlive(connFd); struct timeval to={1, 0}; - taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + if (ret != 0) { + taosCloseSocket(connFd); + tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), + taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); + continue; + } + // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 5721525adee3fc847a1ba2476ccb0995fb50a65c..faa6d40da398045e43dba3bbdbaf9ef6c7ccb1ff 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -188,7 +188,8 @@ int main(int argc, char *argv[]) { tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads); tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize); - getchar(); + int ch = getchar(); + UNUSED(ch); taosCloseLog(); diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index 60271c771ca0a01bd449cb878fe2269759250fd3..aa38a56f38146bafda8a10ca786bf085bb4ea339 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -5,12 +5,12 @@ INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) IF (TD_LINUX) - LIST(REMOVE_ITEM SRC src/tarbitrator.c) + LIST(REMOVE_ITEM SRC src/syncArbitrator.c) ADD_LIBRARY(sync ${SRC}) TARGET_LINK_LIBRARIES(sync tutil pthread common) - LIST(APPEND BIN_SRC src/tarbitrator.c) - LIST(APPEND BIN_SRC src/taosTcpPool.c) + LIST(APPEND BIN_SRC src/syncArbitrator.c) + LIST(APPEND BIN_SRC src/syncTcp.c) ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 6d0c52284f7f2429ccfa20389ccd66b2447610b9..47090cfa0cc6997a20c6803b8fc299f08e407bb5 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -13,12 +13,14 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_SYNCINT_H -#define TDENGINE_SYNCINT_H +#ifndef TDENGINE_SYNC_INT_H +#define TDENGINE_SYNC_INT_H #ifdef __cplusplus extern "C" { #endif +#include "syncMsg.h" +#include "twal.h" #define sFatal(...) { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); }} #define sError(...) { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); }} @@ -27,83 +29,22 @@ extern "C" { #define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} #define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} -typedef enum { - TAOS_SMSG_SYNC_DATA = 1, - TAOS_SMSG_FORWARD = 2, - TAOS_SMSG_FORWARD_RSP = 3, - TAOS_SMSG_SYNC_REQ = 4, - TAOS_SMSG_SYNC_RSP = 5, - TAOS_SMSG_SYNC_MUST = 6, - TAOS_SMSG_STATUS = 7, - TAOS_SMSG_SYNC_DATA_RSP = 8, -} ESyncMsgType; +#define SYNC_TCP_THREADS 2 +#define SYNC_MAX_NUM 2 #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) #define SYNC_RECV_BUFFER_SIZE (5*1024*1024) -#define SYNC_FWD_TIMER 300 -#define SYNC_ROLE_TIMER 10000 -#define SYNC_WAIT_AFTER_CHOOSE_MASTER 3 + +#define SYNC_MAX_FWDS 512 +#define SYNC_FWD_TIMER 300 +#define SYNC_ROLE_TIMER 15000 // ms +#define SYNC_CHECK_INTERVAL 1000 // ms +#define SYNC_WAIT_AFTER_CHOOSE_MASTER 10 // ms #define nodeRole pNode->peerInfo[pNode->selfIndex]->role #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version #define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus -#pragma pack(push, 1) - -typedef struct { - char type; // msg type - char pversion; // protocol version - char reserved[6]; // not used - int32_t vgId; // vg ID - int32_t len; // content length, does not include head - // char cont[]; // message content starts from here -} SSyncHead; - -typedef struct { - SSyncHead syncHead; - uint16_t port; - char fqdn[TSDB_FQDN_LEN]; - int32_t sourceId; // only for arbitrator -} SFirstPkt; - -typedef struct { - int8_t sync; -} SFirstPktRsp; - -typedef struct { - int8_t role; - uint64_t version; -} SPeerStatus; - -typedef struct { - int8_t role; - int8_t ack; - int8_t type; - int8_t reserved[3]; - uint16_t tranId; - uint64_t version; - SPeerStatus peersStatus[]; -} SPeersStatus; - -typedef struct { - char name[TSDB_FILENAME_LEN]; - uint32_t magic; - uint32_t index; - uint64_t fversion; - int64_t size; -} SFileInfo; - -typedef struct { - int8_t sync; -} SFileAck; - -typedef struct { - uint64_t version; - int32_t code; -} SFwdRsp; - -#pragma pack(pop) - typedef struct { char * buffer; int32_t bufferSize; @@ -145,9 +86,10 @@ typedef struct SsyncPeer { int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process + int32_t refCount; + int64_t rid; void * timer; void * pConn; - int32_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -157,6 +99,7 @@ typedef struct SSyncNode { int8_t quorum; int8_t selfIndex; uint32_t vgId; + int32_t refCount; int64_t rid; SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator SSyncPeer * pMaster; @@ -180,13 +123,13 @@ extern int32_t tsSyncNum; extern char tsNodeFqdn[TSDB_FQDN_LEN]; extern char * syncStatus[]; -void *syncRetrieveData(void *param); -void *syncRestoreData(void *param); -int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); -void syncRestartConnection(SSyncPeer *pPeer); -void syncBroadcastStatus(SSyncNode *pNode); -void syncAddPeerRef(SSyncPeer *pPeer); -int32_t syncDecPeerRef(SSyncPeer *pPeer); +void * syncRetrieveData(void *param); +void * syncRestoreData(void *param); +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); +void syncRestartConnection(SSyncPeer *pPeer); +void syncBroadcastStatus(SSyncNode *pNode); +SSyncPeer *syncAcquirePeer(int64_t rid); +void syncReleasePeer(SSyncPeer *pPeer); #ifdef __cplusplus } diff --git a/src/sync/inc/syncMsg.h b/src/sync/inc/syncMsg.h new file mode 100644 index 0000000000000000000000000000000000000000..73f4223c882ff9ce544984b37843070cf579c44a --- /dev/null +++ b/src/sync/inc/syncMsg.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_SYNC_MSG_H +#define TDENGINE_SYNC_MSG_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "tsync.h" + +typedef enum { + TAOS_SMSG_START = 0, + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_SYNC_DATA_RSP = 2, + TAOS_SMSG_SYNC_FWD = 3, + TAOS_SMSG_SYNC_FWD_RSP = 4, + TAOS_SMSG_SYNC_REQ = 5, + TAOS_SMSG_SYNC_REQ_RSP = 6, + TAOS_SMSG_SYNC_MUST = 7, + TAOS_SMSG_SYNC_MUST_RSP = 8, + TAOS_SMSG_STATUS = 9, + TAOS_SMSG_STATUS_RSP = 10, + TAOS_SMSG_SETUP = 11, + TAOS_SMSG_SETUP_RSP = 12, + TAOS_SMSG_SYNC_FILE = 13, + TAOS_SMSG_SYNC_FILE_RSP = 14, + TAOS_SMSG_END = 15, +} ESyncMsgType; + +typedef enum { + SYNC_STATUS_BROADCAST, + SYNC_STATUS_BROADCAST_RSP, + SYNC_STATUS_SETUP_CONN, + SYNC_STATUS_SETUP_CONN_RSP, + SYNC_STATUS_EXCHANGE_DATA, + SYNC_STATUS_EXCHANGE_DATA_RSP, + SYNC_STATUS_CHECK_ROLE, + SYNC_STATUS_CHECK_ROLE_RSP +} ESyncStatusType; + +#pragma pack(push, 1) + +typedef struct { + int8_t type; // msg type + int8_t protocol; // protocol version + uint16_t signature; // fixed value + int32_t code; // + int32_t cId; // cluster Id + int32_t vgId; // vg ID + int32_t len; // content length, does not include head + uint32_t cksum; +} SSyncHead; + +typedef struct { + SSyncHead head; + uint16_t port; + uint16_t tranId; + int32_t sourceId; // only for arbitrator + char fqdn[TSDB_FQDN_LEN]; +} SSyncMsg; + +typedef struct { + SSyncHead head; + int8_t sync; + int8_t reserved; + uint16_t tranId; + int8_t reserverd[4]; +} SSyncRsp; + +typedef struct { + int8_t role; + uint64_t version; +} SPeerStatus; + +typedef struct { + SSyncHead head; + int8_t role; + int8_t ack; + int8_t type; + int8_t reserved[3]; + uint16_t tranId; + uint64_t version; + SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; +} SPeersStatus; + +typedef struct { + SSyncHead head; + char name[TSDB_FILENAME_LEN]; + uint32_t magic; + uint32_t index; + uint64_t fversion; + int64_t size; +} SFileInfo; + +typedef struct { + SSyncHead head; + int8_t sync; +} SFileAck; + +typedef struct { + SSyncHead head; + uint64_t version; + int32_t code; +} SFwdRsp; + +#pragma pack(pop) + +#define SYNC_PROTOCOL_VERSION 1 +#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) + +extern char *statusType[]; + +uint16_t syncGenTranId(); +int32_t syncCheckHead(SSyncHead *pHead); + +void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); +void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); +void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); + +void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); +void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_VNODEPEER_H diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/syncTcp.h similarity index 69% rename from src/sync/inc/taosTcpPool.h rename to src/sync/inc/syncTcp.h index 41043b0cd4c886616d5cecd2739eae684052c395..d4674fee6bd07d5ab68284399d9a6502dbd5ec2c 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/syncTcp.h @@ -13,30 +13,27 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TCP_POOL_H -#define TDENGINE_TCP_POOL_H +#ifndef TDENGINE_SYNC_TCP_POOL_H +#define TDENGINE_SYNC_TCP_POOL_H #ifdef __cplusplus extern "C" { #endif -typedef void *ttpool_h; -typedef void *tthread_h; - typedef struct { int32_t numOfThreads; uint32_t serverIp; int16_t port; int32_t bufferSize; - void (*processBrokenLink)(void *ahandle); - int32_t (*processIncomingMsg)(void *ahandle, void *buffer); + void (*processBrokenLink)(int64_t handleId); + int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); void (*processIncomingConn)(int32_t fd, uint32_t ip); } SPoolInfo; -ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo); -void taosCloseTcpThreadPool(ttpool_h); -void * taosAllocateTcpConn(void *, void *ahandle, int32_t connFd); -void taosFreeTcpConn(void *); +void *syncOpenTcpThreadPool(SPoolInfo *pInfo); +void syncCloseTcpThreadPool(void *); +void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); +void syncFreeTcpConn(void *); #ifdef __cplusplus } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/syncArbitrator.c similarity index 79% rename from src/sync/src/tarbitrator.c rename to src/sync/src/syncArbitrator.c index 4016042de2135f732dfeb2bbc3b0fdc65b1b63f6..fed0774346693fbd2d5ef725bfa620433664cee8 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -22,17 +22,17 @@ #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" -#include "taosTcpPool.h" #include "twal.h" #include "tsync.h" #include "syncInt.h" +#include "syncTcp.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); -static void arbProcessBrokenLink(void *param); -static int32_t arbProcessPeerMsg(void *param, void *buffer); -static tsem_t tsArbSem; -static ttpool_h tsArbTcpPool; +static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void arbProcessBrokenLink(int64_t rid); +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); +static tsem_t tsArbSem; +static void * tsArbTcpPool; typedef struct { char id[TSDB_EP_LEN + 24]; @@ -90,7 +90,7 @@ int32_t main(int32_t argc, char *argv[]) { info.processBrokenLink = arbProcessBrokenLink; info.processIncomingMsg = arbProcessPeerMsg; info.processIncomingConn = arbProcessIncommingConnection; - tsArbTcpPool = taosOpenTcpThreadPool(&info); + tsArbTcpPool = syncOpenTcpThreadPool(&info); if (tsArbTcpPool == NULL) { sDebug("failed to open TCP thread pool, exit..."); @@ -101,8 +101,8 @@ int32_t main(int32_t argc, char *argv[]) { tsem_wait(&tsArbSem); - taosCloseTcpThreadPool(tsArbTcpPool); - sInfo("TAOS arbitrator is shut down\n"); + syncCloseTcpThreadPool(tsArbTcpPool); + sInfo("TAOS arbitrator is shut down"); closelog(); return 0; @@ -113,9 +113,9 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); - SFirstPkt firstPkt; - if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { - sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno)); + SSyncMsg msg; + if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sError("failed to read peer sync msg from ip:%s since %s", ipstr, strerror(errno)); taosCloseSocket(connFd); return; } @@ -127,9 +127,9 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } - firstPkt.fqdn[sizeof(firstPkt.fqdn) - 1] = 0; - snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); - if (firstPkt.syncHead.vgId) { + msg.fqdn[TSDB_FQDN_LEN - 1] = 0; + snprintf(pNode->id, sizeof(pNode->id), "vgId:%d, peer:%s:%d", msg.sourceId, msg.fqdn, msg.port); + if (msg.head.vgId) { sDebug("%s, vgId in head is not zero, close the connection", pNode->id); tfree(pNode); taosCloseSocket(connFd); @@ -138,26 +138,26 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { sDebug("%s, arbitrator request is accepted", pNode->id); pNode->nodeFd = connFd; - pNode->pConn = taosAllocateTcpConn(tsArbTcpPool, pNode, connFd); + pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd); return; } -static void arbProcessBrokenLink(void *param) { - SNodeConn *pNode = param; +static void arbProcessBrokenLink(int64_t rid) { + SNodeConn *pNode = (SNodeConn *)rid; sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno)); tfree(pNode); } -static int32_t arbProcessPeerMsg(void *param, void *buffer) { - SNodeConn *pNode = param; +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { + SNodeConn *pNode = (SNodeConn *)rid; SSyncHead head; int32_t bytes = 0; char * cont = (char *)buffer; - int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head)); - if (hlen != sizeof(head)) { + int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(SSyncHead)); + if (hlen != sizeof(SSyncHead)) { sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen); return -1; } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 3fa6323f4d073eaed32c726d51709bb54034aa68..436f4de0988aa87836172b8c0284ca767d27c5d8 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -23,47 +23,46 @@ #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" -#include "taosTcpPool.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" +#include "syncTcp.h" #include "syncInt.h" -// global configurable -int32_t tsMaxSyncNum = 2; -int32_t tsSyncTcpThreads = 2; -int32_t tsMaxWatchFiles = 500; -int32_t tsMaxFwdInfo = 200; -int32_t tsSyncTimer = 1; +int32_t tsSyncNum = 0; // number of sync in process in whole system +char tsNodeFqdn[TSDB_FQDN_LEN] = {0}; -// module global, not configurable -int32_t tsSyncNum; // number of sync in process in whole system -char tsNodeFqdn[TSDB_FQDN_LEN]; - -static ttpool_h tsTcpPool; -static void * tsSyncTmrCtrl = NULL; -static void * tsVgIdHash; -static int32_t tsSyncRefId = -1; +static void * tsTcpPool = NULL; +static void * tsSyncTmrCtrl = NULL; +static void * tsVgIdHash = NULL; +static int32_t tsNodeRefId = -1; +static int32_t tsPeerRefId = -1; // local functions static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); static void syncRecoverFromMaster(SSyncPeer *pPeer); static void syncCheckPeerConnection(void *param, void *tmrId); -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); -static void syncProcessBrokenLink(void *param); -static int32_t syncProcessPeerMsg(void *param, void *buffer); +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); +static void syncProcessBrokenLink(int64_t rid); +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); +static void syncFreePeer(void *); static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); static void syncMonitorFwdInfos(void *param, void *tmrId); static void syncMonitorNodeRole(void *param, void *tmrId); static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); +static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); static void syncRestartPeer(SSyncPeer *pPeer); static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp); + static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); +static void syncStartCheckPeerConn(SSyncPeer *pPeer); +static void syncStopCheckPeerConn(SSyncPeer *pPeer); +static SSyncNode *syncAcquireNode(int64_t rid); +static void syncReleaseNode(SSyncNode *pNode); char* syncRole[] = { "offline", @@ -81,36 +80,10 @@ char *syncStatus[] = { "invalid" }; -typedef enum { - SYNC_STATUS_BROADCAST, - SYNC_STATUS_BROADCAST_RSP, - SYNC_STATUS_SETUP_CONN, - SYNC_STATUS_SETUP_CONN_RSP, - SYNC_STATUS_EXCHANGE_DATA, - SYNC_STATUS_EXCHANGE_DATA_RSP, - SYNC_STATUS_CHECK_ROLE, - SYNC_STATUS_CHECK_ROLE_RSP -} ESyncStatusType; - -char *statusType[] = { - "broadcast", - "broadcast-rsp", - "setup-conn", - "setup-conn-rsp", - "exchange-data", - "exchange-data-rsp", - "check-role", - "check-role-rsp" -}; - -uint16_t syncGenTranId() { - return taosRand() & 0XFFFF; -} - int32_t syncInit() { SPoolInfo info = {0}; - info.numOfThreads = tsSyncTcpThreads; + info.numOfThreads = SYNC_TCP_THREADS; info.serverIp = 0; info.port = tsSyncPort; info.bufferSize = SYNC_MAX_SIZE; @@ -118,32 +91,37 @@ int32_t syncInit() { info.processIncomingMsg = syncProcessPeerMsg; info.processIncomingConn = syncProcessIncommingConnection; - tsTcpPool = taosOpenTcpThreadPool(&info); + tsTcpPool = syncOpenTcpThreadPool(&info); if (tsTcpPool == NULL) { sError("failed to init tcpPool"); + syncCleanUp(); return -1; } tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); if (tsSyncTmrCtrl == NULL) { sError("failed to init tmrCtrl"); - taosCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; + syncCleanUp(); return -1; } tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVgIdHash == NULL) { - sError("failed to init tsVgIdHash"); - taosTmrCleanUp(tsSyncTmrCtrl); - taosCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; - tsSyncTmrCtrl = NULL; + sError("failed to init vgIdHash"); + syncCleanUp(); return -1; } - tsSyncRefId = taosOpenRef(200, syncFreeNode); - if (tsSyncRefId < 0) { + tsNodeRefId = taosOpenRef(200, syncFreeNode); + if (tsNodeRefId < 0) { + sError("failed to init node ref"); + syncCleanUp(); + return -1; + } + + tsPeerRefId = taosOpenRef(1000, syncFreePeer); + if (tsPeerRefId < 0) { + sError("failed to init peer ref"); syncCleanUp(); return -1; } @@ -155,12 +133,12 @@ int32_t syncInit() { } void syncCleanUp() { - if (tsTcpPool) { - taosCloseTcpThreadPool(tsTcpPool); + if (tsTcpPool != NULL) { + syncCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; } - if (tsSyncTmrCtrl) { + if (tsSyncTmrCtrl != NULL) { taosTmrCleanUp(tsSyncTmrCtrl); tsSyncTmrCtrl = NULL; } @@ -170,8 +148,15 @@ void syncCleanUp() { tsVgIdHash = NULL; } - taosCloseRef(tsSyncRefId); - tsSyncRefId = -1; + if (tsNodeRefId != -1) { + taosCloseRef(tsNodeRefId); + tsNodeRefId = -1; + } + + if (tsPeerRefId != -1) { + taosCloseRef(tsPeerRefId); + tsPeerRefId = -1; + } sInfo("sync module is cleaned up"); } @@ -204,7 +189,8 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->quorum = pCfg->quorum; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - pNode->rid = taosAddRef(tsSyncRefId, pNode); + pNode->refCount = 1; + pNode->rid = taosAddRef(tsNodeRefId, pNode); if (pNode->rid < 0) { syncFreeNode(pNode); return -1; @@ -237,7 +223,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]); - pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo * sizeof(SFwdInfo), 1); + pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + SYNC_MAX_FWDS * sizeof(SFwdInfo), 1); if (pNode->pSyncFwds == NULL) { sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId); terrno = TAOS_SYSTEM_ERROR(errno); @@ -266,13 +252,18 @@ int64_t syncStart(const SSyncInfo *pInfo) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); + } + return pNode->rid; } void syncStop(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; sInfo("vgId:%d, cleanup sync", pNode->vgId); @@ -293,14 +284,14 @@ void syncStop(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); - taosRemoveRef(tsSyncRefId, rid); + syncReleaseNode(pNode); + taosRemoveRef(tsNodeRefId, rid); } int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { int32_t i, j; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, @@ -308,6 +299,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { pthread_mutex_lock(&pNode->mutex); + syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStopCheckPeerConn(pNode->peerInfo[index]); + } + for (i = 0; i < pNode->replica; ++i) { for (j = 0; j < pNewCfg->replica; ++j) { if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && @@ -337,6 +333,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { newPeers[i] = pNode->peerInfo[j]; } + if (newPeers[i] == NULL) { + sError("vgId:%d, failed to reconfig", pNode->vgId); + return TSDB_CODE_SYN_INVALID_CONFIG; + } + if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) { pNode->selfIndex = i; } @@ -359,61 +360,55 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); + } + pthread_mutex_unlock(&pNode->mutex); sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); syncBroadcastStatus(pNode); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); return 0; } int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); - if (pNode == NULL) return 0; + SSyncNode *pNode = syncAcquireNode(rid); + if (pNode == NULL) return 0; int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); - taosReleaseRef(tsSyncRefId, rid); - + syncReleaseNode(pNode); return code; } void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncPeer *pPeer = pNode->pMaster; if (pPeer && pNode->quorum > 1) { - char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0}; - - SSyncHead *pHead = (SSyncHead *)msg; - pHead->type = TAOS_SMSG_FORWARD_RSP; - pHead->len = sizeof(SFwdRsp); - - SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead)); - pFwdRsp->version = version; - pFwdRsp->code = code; - - int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); - int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, msgLen); + SFwdRsp rsp; + syncBuildSyncFwdRsp(&rsp, pNode->vgId, version, code); - if (retLen == msgLen) { - sTrace("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version); + if (taosWriteMsg(pPeer->peerFd, &rsp, sizeof(SFwdRsp)) == sizeof(SFwdRsp)) { + sTrace("%s, forward-rsp is sent, code:0x%x hver:%" PRIu64, pPeer->id, code, version); } else { - sDebug("%s, failed to send forward ack, restart", pPeer->id); + sDebug("%s, failed to send forward-rsp, restart", pPeer->id); syncRestartConnection(pPeer); } } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #if 0 void syncRecover(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; // to do: add a few lines to check if recover is OK @@ -434,12 +429,12 @@ void syncRecover(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #endif int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return -1; pNodesRole->selfIndex = pNode->selfIndex; @@ -448,8 +443,7 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { pNodesRole->role[i] = pNode->peerInfo[i]->role; } - taosReleaseRef(tsSyncRefId, rid); - + syncReleaseNode(pNode); return 0; } @@ -484,6 +478,7 @@ static void syncAddArbitrator(SSyncNode *pNode) { static void syncFreeNode(void *param) { SSyncNode *pNode = param; + sDebug("vgId:%d, node is freed, refCount:%d", pNode->vgId, pNode->refCount); pthread_mutex_destroy(&pNode->mutex); tfree(pNode->pRecv); @@ -491,18 +486,50 @@ static void syncFreeNode(void *param) { tfree(pNode); } -void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); } +SSyncNode *syncAcquireNode(int64_t rid) { + SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid); + if (pNode == NULL) { + sDebug("failed to acquire node from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, acquire node refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount); + } -int32_t syncDecPeerRef(SSyncPeer *pPeer) { - if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) { - taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); + return pNode; +} - sDebug("%s, resource is freed", pPeer->id); - tfree(pPeer); - return 0; +void syncReleaseNode(SSyncNode *pNode) { + int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, release node refId:%" PRId64 ", refCount:%d", pNode->vgId, pNode->rid, refCount); + + taosReleaseRef(tsNodeRefId, pNode->rid); +} + +static void syncFreePeer(void *param) { + SSyncPeer *pPeer = param; + sDebug("%s, peer is freed, refCount:%d", pPeer->id, pPeer->refCount); + + syncReleaseNode(pPeer->pSyncNode); + tfree(pPeer); +} + +SSyncPeer *syncAcquirePeer(int64_t rid) { + SSyncPeer *pPeer = taosAcquireRef(tsPeerRefId, rid); + if (pPeer == NULL) { + sDebug("failed to acquire peer from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pPeer->refCount, 1); + sTrace("%s, acquire peer refId:%" PRId64 ", refCount:%d", pPeer->id, rid, refCount); } - return 1; + return pPeer; +} + +void syncReleasePeer(SSyncPeer *pPeer) { + int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1); + sTrace("%s, release peer refId:%" PRId64 ", refCount:%d", pPeer->id, pPeer->rid, refCount); + + taosReleaseRef(tsPeerRefId, pPeer->rid); } static void syncClosePeerConn(SSyncPeer *pPeer) { @@ -512,7 +539,8 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { taosClose(pPeer->syncFd); if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; - taosFreeTcpConn(pPeer->pConn); + void *pConn = pPeer->pConn; + if (pConn != NULL) syncFreeTcpConn(pPeer->pConn); } } @@ -521,11 +549,32 @@ static void syncRemovePeer(SSyncPeer *pPeer) { pPeer->ip = 0; syncClosePeerConn(pPeer); - syncDecPeerRef(pPeer); + //taosRemoveRef(tsPeerRefId, pPeer->rid); + syncReleasePeer(pPeer); +} + +static void syncStartCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; + + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); + if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { + int32_t checkMs = 100 + (pNode->vgId * 10) % 100; + if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; + sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); + taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); + } +} + +static void syncStopCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + + taosTmrStopA(&pPeer->timer); + sDebug("%s, stop check peer connection", pPeer->id); } static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { - uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn); + uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ip == 0xFFFFFFFF) { sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno)); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -547,17 +596,11 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->role = TAOS_SYNC_ROLE_OFFLINE; pPeer->pSyncNode = pNode; pPeer->refCount = 1; + pPeer->rid = taosAddRef(tsPeerRefId, pPeer); - sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port); - int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { - int32_t checkMs = 100 + (pNode->vgId * 10) % 100; - if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; - sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); - taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer); - } + sInfo("%s, %p it is configured, ep:%s:%u rid:%" PRId64, pPeer->id, pPeer, pPeer->fqdn, pPeer->port, pPeer->rid); - taosAcquireRef(tsSyncRefId, pNode->rid); + (void)syncAcquireNode(pNode->rid); return pPeer; } @@ -599,6 +642,9 @@ static void syncChooseMaster(SSyncNode *pNode) { index = i; } } + sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index); + } else { + sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica); } // add arbitrator connection @@ -619,6 +665,11 @@ static void syncChooseMaster(SSyncNode *pNode) { } } } + + if (index >= 0) { + sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId, + pNode->peerInfo[index]->id, index, onlineNum, replica); + } } if (index >= 0) { @@ -660,9 +711,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; + if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) { + sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } else { + nodeRole = TAOS_SYNC_ROLE_UNSYNCED; + sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } (*pNode->notifyRole)(pNode->vgId, nodeRole); - sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { for (int32_t index = 0; index < pNode->replica; ++index) { @@ -717,7 +772,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new if (pMaster) { // master is there pNode->pMaster = pMaster; - sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); + sDebug("%s, it is the master, replica:%d sver:%" PRIu64, pMaster->id, pNode->replica, pMaster->version); if (syncValidateMaster(pPeer) < 0) return; @@ -750,10 +805,10 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } if (consistent) { - sDebug("vgId:%d, choose master", pNode->vgId); + sDebug("vgId:%d, choose master, replica:%d", pNode->vgId, pNode->replica); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inequality, replica:%d", pNode->vgId, pNode->replica); } } @@ -782,7 +837,7 @@ static void syncRestartPeer(SSyncPeer *pPeer) { int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { sDebug("%s, check peer connection in 1000 ms", pPeer->id); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } @@ -795,7 +850,7 @@ void syncRestartConnection(SSyncPeer *pPeer) { static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - sDebug("%s, sync-req is received", pPeer->id); + sInfo("%s, sync-req is received", pPeer->id); if (pPeer->ip == 0) return; @@ -811,25 +866,30 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { } // start a new thread to retrieve the data - syncAddPeerRef(pPeer); + (void)syncAcquirePeer(pPeer->rid); + pthread_attr_t thattr; pthread_t thread; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); + int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); - if (ret != 0) { - sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno)); - syncDecPeerRef(pPeer); + if (ret < 0) { + sError("%s, failed to create sync retrieve thread since %s", pPeer->id, strerror(errno)); + syncReleasePeer(pPeer); } else { pPeer->sstatus = TAOS_SYNC_STATUS_START; - sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + sDebug("%s, sync retrieve thread:0x%08" PRIx64 " create successfully, rid:%" PRId64 ", set sstatus:%s", pPeer->id, + taosGetPthreadId(thread), pPeer->rid, syncStatus[pPeer->sstatus]); } } static void syncNotStarted(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -838,15 +898,22 @@ static void syncNotStarted(void *param, void *tmrId) { sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncTryRecoverFromMaster(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncRecoverFromMaster(SSyncPeer *pPeer) { @@ -860,34 +927,28 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); // Ensure the sync of mnode not interrupted - if (pNode->vgId != 1 && tsSyncNum >= tsMaxSyncNum) { + if (pNode->vgId != 1 && tsSyncNum >= SYNC_MAX_NUM) { sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum); - taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } sDebug("%s, try to sync", pPeer->id); - SFirstPkt firstPkt; - memset(&firstPkt, 0, sizeof(firstPkt)); - firstPkt.syncHead.type = TAOS_SMSG_SYNC_REQ; - firstPkt.syncHead.vgId = pNode->vgId; - firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead); - tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); - firstPkt.port = tsSyncPort; - taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + SSyncMsg msg; + syncBuildSyncReqMsg(&msg, pNode->vgId); - if (taosWriteMsg(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { + taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); + + if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { sError("%s, failed to send sync-req to peer", pPeer->id); } else { - nodeSStatus = TAOS_SYNC_STATUS_START; - sInfo("%s, sync-req is sent to peer, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, sync-req is sent to peer, tranId:%u, sstatus:%s", pPeer->id, msg.tranId, syncStatus[nodeSStatus]); } } -static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { +static void syncProcessFwdResponse(SFwdRsp *pFwdRsp, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFwdRsp * pFwdRsp = (SFwdRsp *)cont; SSyncFwds *pSyncFwds = pNode->pSyncFwds; SFwdInfo * pFwdInfo; @@ -897,18 +958,19 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) { // find the forwardInfo from first for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { - pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % tsMaxFwdInfo; - if (pFwdRsp->version == pFwdInfo->version) break; + pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % SYNC_MAX_FWDS; + if (pFwdRsp->version == pFwdInfo->version) { + syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code); + syncRemoveConfirmedFwdInfo(pNode); + return; + } } - - syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code); - syncRemoveConfirmedFwdInfo(pNode); } } static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SWalHead * pHead = (SWalHead *)cont; + SWalHead * pHead = (SWalHead *)(cont + sizeof(SSyncHead)); sTrace("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); @@ -925,9 +987,8 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { } } -static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { - SSyncNode * pNode = pPeer->pSyncNode; - SPeersStatus *pPeersStatus = (SPeersStatus *)cont; +static void syncProcessPeersStatusMsg(SPeersStatus *pPeersStatus, SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; sDebug("%s, status is received, self:%s:%s:%" PRIu64 ", peer:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role], @@ -941,23 +1002,22 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { } } -static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { +static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead) { if (pPeer->peerFd < 0) return -1; int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead)); if (hlen != sizeof(SSyncHead)) { - sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen); + sDebug("%s, failed to read msg since %s, hlen:%d", pPeer->id, tstrerror(errno), hlen); return -1; } - // head.len = htonl(head.len); - if (pHead->len < 0) { - sError("%s, invalid pkt length, hlen:%d", pPeer->id, pHead->len); + int32_t code = syncCheckHead(pHead); + if (code != 0) { + sError("%s, failed to check msg head since %s, type:%d", pPeer->id, tstrerror(code), pHead->type); return -1; } - assert(pHead->len <= TSDB_MAX_WAL_SIZE); - int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); + int32_t bytes = taosReadMsg(pPeer->peerFd, (char *)pHead + sizeof(SSyncHead), pHead->len); if (bytes != pHead->len) { sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len); return -1; @@ -966,67 +1026,68 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return 0; } -static int32_t syncProcessPeerMsg(void *param, void *buffer) { - SSyncPeer *pPeer = param; - SSyncHead head; - char * cont = buffer; +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return -1; + SSyncHead *pHead = buffer; SSyncNode *pNode = pPeer->pSyncNode; + pthread_mutex_lock(&pNode->mutex); - int32_t code = syncReadPeerMsg(pPeer, &head, cont); + int32_t code = syncReadPeerMsg(pPeer, pHead); if (code == 0) { - if (head.type == TAOS_SMSG_FORWARD) { - syncProcessForwardFromPeer(cont, pPeer); - } else if (head.type == TAOS_SMSG_FORWARD_RSP) { - syncProcessFwdResponse(cont, pPeer); - } else if (head.type == TAOS_SMSG_SYNC_REQ) { - syncProcessSyncRequest(cont, pPeer); - } else if (head.type == TAOS_SMSG_STATUS) { - syncProcessPeersStatusMsg(cont, pPeer); + if (pHead->type == TAOS_SMSG_SYNC_FWD) { + syncProcessForwardFromPeer(buffer, pPeer); + } else if (pHead->type == TAOS_SMSG_SYNC_FWD_RSP) { + syncProcessFwdResponse(buffer, pPeer); + } else if (pHead->type == TAOS_SMSG_SYNC_REQ) { + syncProcessSyncRequest(buffer, pPeer); + } else if (pHead->type == TAOS_SMSG_STATUS) { + syncProcessPeersStatusMsg(buffer, pPeer); } } pthread_mutex_unlock(&pNode->mutex); + syncReleasePeer(pPeer); return code; } -#define statusMsgLen sizeof(SSyncHead) + sizeof(SPeersStatus) + sizeof(SPeerStatus) * TAOS_SYNC_MAX_REPLICA - -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { - SSyncNode *pNode = pPeer->pSyncNode; - char msg[statusMsgLen] = {0}; - - if (pPeer->peerFd < 0 || pPeer->ip == 0) return; +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { + if (pPeer->peerFd < 0 || pPeer->ip == 0) { + sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd); + syncRestartConnection(pPeer); + return -1; + } - SSyncHead * pHead = (SSyncHead *)msg; - SPeersStatus *pPeersStatus = (SPeersStatus *)(msg + sizeof(SSyncHead)); + SSyncNode * pNode = pPeer->pSyncNode; + SPeersStatus msg; - pHead->type = TAOS_SMSG_STATUS; - pHead->len = statusMsgLen - sizeof(SSyncHead); + memset(&msg, 0, sizeof(SPeersStatus)); + syncBuildPeersStatus(&msg, pNode->vgId); - pPeersStatus->version = nodeVersion; - pPeersStatus->role = nodeRole; - pPeersStatus->ack = ack; - pPeersStatus->type = type; - pPeersStatus->tranId = tranId; + msg.role = nodeRole; + msg.ack = ack; + msg.type = type; + msg.tranId = tranId; + msg.version = nodeVersion; for (int32_t i = 0; i < pNode->replica; ++i) { - pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role; - pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version; + msg.peersStatus[i].role = pNode->peerInfo[i]->role; + msg.peersStatus[i].version = pNode->peerInfo[i]->version; } - int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); - if (retLen == statusMsgLen) { + if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SPeersStatus)) == sizeof(SPeersStatus)) { sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], - syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId, - statusType[pPeersStatus->type], pPeer->peerFd); + syncStatus[pPeer->sstatus], pPeer->version, ack, tranId, statusType[type], pPeer->peerFd); + return 0; } else { sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); + return -1; } } @@ -1043,33 +1104,30 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } - SFirstPkt firstPkt; - memset(&firstPkt, 0, sizeof(firstPkt)); - firstPkt.syncHead.vgId = pPeer->nodeId ? pNode->vgId : 0; - firstPkt.syncHead.type = TAOS_SMSG_STATUS; - tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); - firstPkt.port = tsSyncPort; - firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId + SSyncMsg msg; + syncBuildSyncSetupMsg(&msg, pPeer->nodeId ? pNode->vgId : 0); - if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { - sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d", pPeer->id, connFd, pPeer->syncFd); + if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) == sizeof(SSyncMsg)) { + sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, msg.tranId); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; - pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); taosClose(connFd); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } static void syncCheckPeerConnection(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -1078,6 +1136,8 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { syncSetupPeerConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { @@ -1088,16 +1148,20 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - syncAddPeerRef(pPeer); - int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); + (void)syncAcquirePeer(pPeer->rid); + + int32_t ret = pthread_create(&thread, &thattr, (void *)syncRestoreData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); if (ret < 0) { - sError("%s, failed to create sync thread", pPeer->id); + SSyncNode *pNode = pPeer->pSyncNode; + nodeSStatus = TAOS_SYNC_STATUS_INIT; + sError("%s, failed to create sync restore thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); + syncReleasePeer(pPeer); } else { - sInfo("%s, sync connection is up", pPeer->id); + sInfo("%s, sync restore thread:0x%08" PRIx64 " create successfully, rid:%" PRId64, pPeer->id, + taosGetPthreadId(thread), pPeer->rid); } } @@ -1108,14 +1172,21 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); - SFirstPkt firstPkt; - if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { - sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno)); + SSyncMsg msg; + if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sError("failed to read peer sync msg from ip:%s since %s", ipstr, strerror(errno)); + taosCloseSocket(connFd); + return; + } + + int32_t code = syncCheckHead((SSyncHead *)(&msg)); + if (code != 0) { + sError("failed to check peer sync msg from ip:%s since %s", ipstr, strerror(code)); taosCloseSocket(connFd); return; } - int32_t vgId = firstPkt.syncHead.vgId; + int32_t vgId = msg.head.vgId; SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { sError("vgId:%d, vgId could not be found", vgId); @@ -1123,31 +1194,35 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } + sDebug("vgId:%d, sync connection is incomming, tranId:%u", vgId, msg.tranId); + SSyncNode *pNode = *ppNode; pthread_mutex_lock(&pNode->mutex); SSyncPeer *pPeer; for (i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; - if (pPeer && (strcmp(pPeer->fqdn, firstPkt.fqdn) == 0) && (pPeer->port == firstPkt.port)) break; + if (pPeer && (strcmp(pPeer->fqdn, msg.fqdn) == 0) && (pPeer->port == msg.port)) break; } pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL; if (pPeer == NULL) { - sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, firstPkt.fqdn, firstPkt.port); + sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, msg.fqdn, msg.port); taosCloseSocket(connFd); // syncSendVpeerCfgMsg(sync); } else { // first packet tells what kind of link - if (firstPkt.syncHead.type == TAOS_SMSG_SYNC_DATA) { + if (msg.head.type == TAOS_SMSG_SYNC_DATA) { pPeer->syncFd = connFd; + nodeSStatus = TAOS_SYNC_STATUS_START; + sInfo("%s, sync-data msg from master is received, tranId:%u, set sstatus:%s", pPeer->id, msg.tranId, + syncStatus[nodeSStatus]); syncCreateRestoreDataThread(pPeer); } else { sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; - pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); sDebug("%s, ready to exchange data", pPeer->id); syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId()); } @@ -1156,36 +1231,36 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pthread_mutex_unlock(&pNode->mutex); } -static void syncProcessBrokenLink(void *param) { - if (param == NULL) return; // the connection for arbitrator - SSyncPeer *pPeer = param; +static void syncProcessBrokenLink(int64_t rid) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&pNode->mutex); sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); pPeer->peerFd = -1; - if (syncDecPeerRef(pPeer) != 0) { - syncRestartConnection(pPeer); - } - + syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, pNode->rid); + + syncReleasePeer(pPeer); } -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { +static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; int64_t time = taosGetTimestampMs(); - if (pSyncFwds->fwds >= tsMaxFwdInfo) { - pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo; - pSyncFwds->fwds--; + if (pSyncFwds->fwds >= SYNC_MAX_FWDS) { + // pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; + // pSyncFwds->fwds--; + sError("vgId:%d, failed to save fwd info, hver:%" PRIu64 " fwds:%d", pNode->vgId, version, pSyncFwds->fwds); + return TSDB_CODE_SYN_TOO_MANY_FWDINFO; } if (pSyncFwds->fwds > 0) { - pSyncFwds->last = (pSyncFwds->last + 1) % tsMaxFwdInfo; + pSyncFwds->last = (pSyncFwds->last + 1) % SYNC_MAX_FWDS; } SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last; @@ -1196,6 +1271,8 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { pSyncFwds->fwds++; sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); + + return 0; } static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { @@ -1206,11 +1283,10 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; if (pFwdInfo->confirmed == 0) break; - pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo; + pSyncFwds->first = (pSyncFwds->first + 1) % SYNC_MAX_FWDS; pSyncFwds->fwds--; if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last; - // sDebug("vgId:%d, fwd info is removed, hver:%d, fwds:%d", - // pNode->vgId, pFwdInfo->version, pSyncFwds->fwds); + sTrace("vgId:%d, fwd info is removed, hver:%" PRIu64 " fwds:%d", pNode->vgId, pFwdInfo->version, pSyncFwds->fwds); memset(pFwdInfo, 0, sizeof(SFwdInfo)); } } @@ -1232,7 +1308,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } if (confirm && pFwdInfo->confirmed == 0) { - sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); + sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:0x%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); (*pNode->confirmForward)(pNode->vgId, pFwdInfo->mhandle, pFwdInfo->code); pFwdInfo->confirmed = 1; } @@ -1240,7 +1316,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code static void syncMonitorNodeRole(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; for (int32_t index = 0; index < pNode->replica; index++) { @@ -1257,12 +1333,12 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { } pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static void syncMonitorFwdInfos(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncFwds *pSyncFwds = pNode->pSyncFwds; @@ -1273,7 +1349,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { if (pSyncFwds->fwds > 0) { pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { - SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; + SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % SYNC_MAX_FWDS; if (ABS(time - pFwdInfo->time) < 2000) break; sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, @@ -1288,7 +1364,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) { @@ -1308,7 +1384,9 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } } - return TSDB_CODE_SYN_INVALID_VERSION; + if (pNode->replica != 1) { + return TSDB_CODE_SYN_INVALID_VERSION; + } } // always update version @@ -1318,14 +1396,12 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; - // only pkt from RPC or CQ can be forwarded + // only msg from RPC or CQ can be forwarded if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; // a hacker way to improve the performance pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); - pSyncHead->type = TAOS_SMSG_FORWARD; - pSyncHead->pversion = 0; - pSyncHead->len = sizeof(SWalHead) + pWalHead->len; + syncBuildSyncFwdMsg(pSyncHead, pNode->vgId, sizeof(SWalHead) + pWalHead->len); fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head pthread_mutex_lock(&pNode->mutex); @@ -1336,8 +1412,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; if (pNode->quorum > 1 && code == 0) { - syncSaveFwdInfo(pNode, pWalHead->version, mhandle); - code = 1; + code = syncSaveFwdInfo(pNode, pWalHead->version, mhandle); + if (code >= 0) code = 1; } int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); @@ -1355,4 +1431,3 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle return code; } - diff --git a/src/sync/src/syncMsg.c b/src/sync/src/syncMsg.c new file mode 100644 index 0000000000000000000000000000000000000000..034f9a98a70c7373b74a84d24d478b52d7bf4df9 --- /dev/null +++ b/src/sync/src/syncMsg.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" +#include "tchecksum.h" +#include "syncInt.h" + +char *statusType[] = { + "broadcast", + "broadcast-rsp", + "setup-conn", + "setup-conn-rsp", + "exchange-data", + "exchange-data-rsp", + "check-role", + "check-role-rsp" +}; + +uint16_t syncGenTranId() { + return taosRand() & 0XFFFF; +} + +static void syncBuildHead(SSyncHead *pHead) { + pHead->protocol = SYNC_PROTOCOL_VERSION; + pHead->signature = SYNC_SIGNATURE; + pHead->code = 0; + pHead->cId = 0; + taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); +} + +int32_t syncCheckHead(SSyncHead *pHead) { + if (pHead->protocol != SYNC_PROTOCOL_VERSION) return TSDB_CODE_SYN_MISMATCHED_PROTOCOL; + if (pHead->signature != SYNC_SIGNATURE) return TSDB_CODE_SYN_MISMATCHED_SIGNATURE; + if (pHead->cId != 0) return TSDB_CODE_SYN_MISMATCHED_CLUSTERID; + if (pHead->len <= 0 || pHead->len > TSDB_MAX_WAL_SIZE) return TSDB_CODE_SYN_INVALID_MSGLEN; + if (pHead->type <= TAOS_SMSG_START || pHead->type >= TAOS_SMSG_END) return TSDB_CODE_SYN_INVALID_MSGTYPE; + if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SSyncHead))) return TSDB_CODE_SYN_INVALID_CHECKSUM; + + return TSDB_CODE_SUCCESS; +} + +void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len) { + pHead->type = TAOS_SMSG_SYNC_FWD; + pHead->vgId = vgId; + pHead->len = len; + syncBuildHead(pHead); +} + +void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code) { + pMsg->head.type = TAOS_SMSG_SYNC_FWD_RSP; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SFwdRsp) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); + + pMsg->version = version; + pMsg->code = code; +} + +static void syncBuildMsg(SSyncMsg *pMsg, int32_t vgId, ESyncMsgType type) { + pMsg->head.type = type; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SSyncMsg) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); + + pMsg->port = tsSyncPort; + pMsg->tranId = syncGenTranId(); + pMsg->sourceId = vgId; + tstrncpy(pMsg->fqdn, tsNodeFqdn, TSDB_FQDN_LEN); +} + +void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_REQ); } +void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_DATA); } +void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SETUP); } + +void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId) { + pMsg->head.type = TAOS_SMSG_STATUS; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SPeersStatus) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); +} + +void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) { + pMsg->head.type = TAOS_SMSG_SYNC_FILE_RSP; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SFileAck) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); +} + +void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId) { + pMsg->head.type = TAOS_SMSG_SYNC_FILE; + pMsg->head.vgId = vgId; + pMsg->head.len = sizeof(SFileInfo) - sizeof(SSyncHead); + syncBuildHead(&pMsg->head); +} \ No newline at end of file diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 3d262d6e7fa6d092d760df87b7b0e76bbd2e1dba..a5e268cdd262ee1cd4bae6433de9c7c764e6561a 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -36,6 +36,8 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex if (sindex < 0 || eindex < sindex) return; + sDebug("%s, extra files will be removed between sindex:%d and eindex:%d", pPeer->id, sindex, eindex); + while (1) { name[0] = 0; magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion); @@ -54,19 +56,27 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */ SFileInfo sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */ - SFileAck fileAck = {0}; + SFileAck fileAck; memset(&fileAck, 0, sizeof(SFileAck)); int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; uint32_t pindex = 0; // index in last restore bool fileChanged = false; *fversion = 0; - sinfo.index = 0; + sinfo.index = -1; while (1) { // read file info - int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); - if (ret < 0) { - sError("%s, failed to read file info while restore file since %s", pPeer->id, strerror(errno)); + minfo.index = -1; + int32_t ret = taosReadMsg(pPeer->syncFd, &minfo, sizeof(SFileInfo)); + if (ret != sizeof(SFileInfo) || minfo.index == -1) { + sError("%s, failed to read fileinfo while restore file since %s", pPeer->id, strerror(errno)); + break; + } + + assert(ret == sizeof(SFileInfo)); + ret = syncCheckHead((SSyncHead *)(&minfo)); + if (ret != 0) { + sError("%s, failed to check fileinfo while restore file since %s", pPeer->id, strerror(ret)); break; } @@ -75,28 +85,32 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { sDebug("%s, no more files to restore", pPeer->id); // remove extra files after the current index - syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX); + if (sinfo.index != -1) syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX); code = 0; break; } + sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, + minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic); + // remove extra files on slave between the current and last index syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1); pindex = minfo.index; // check the file info sinfo = minfo; - sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size); - sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, - &sinfo.fversion); + sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); + sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, sinfo.name, + sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic); // if file not there or magic is not the same, file shall be synced - memset(&fileAck, 0, sizeof(fileAck)); + memset(&fileAck, 0, sizeof(SFileAck)); + syncBuildFileAck(&fileAck, pNode->vgId); fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1 : 0; // send file ack - ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); - if (ret < 0) { + ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck)); + if (ret != sizeof(SFileAck)) { sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno)); break; } @@ -105,6 +119,8 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { if (fileAck.sync == 0) { sDebug("%s, %s is the same", pPeer->id, minfo.name); continue; + } else { + sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } // if sync is required, open file, receive from master, and write to file @@ -144,7 +160,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { return code; } -static int32_t syncRestoreWal(SSyncPeer *pPeer) { +static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) { SSyncNode *pNode = pPeer->pSyncNode; int32_t ret, code = -1; uint64_t lastVer = 0; @@ -154,7 +170,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); - if (ret < 0) { + if (ret != sizeof(SWalHead)) { sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno)); break; } @@ -166,7 +182,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } // wal sync over ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); - if (ret < 0) { + if (ret != pHead->len) { sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno)); break; } @@ -187,6 +203,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } free(pHead); + *wver = lastVer; return code; } @@ -286,11 +303,12 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { uint64_t fversion = 0; sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - SFirstPktRsp firstPktRsp = {.sync = 1}; - if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { - sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + SSyncRsp rsp = {.sync = 1, .tranId = syncGenTranId()}; + if (taosWriteMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) { + sError("%s, failed to send sync rsp since %s", pPeer->id, strerror(errno)); return -1; } + sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId); sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); int32_t code = syncRestoreFile(pPeer, &fversion); @@ -309,12 +327,19 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeVersion = fversion; - sInfo("%s, start to restore wal", pPeer->id); - if (syncRestoreWal(pPeer) < 0) { - sError("%s, failed to restore wal", pPeer->id); + sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); + uint64_t wver = 0; + code = syncRestoreWal(pPeer, &wver); // lastwar + if (code < 0) { + sError("%s, failed to restore wal, code:%d", pPeer->id, code); return -1; } + if (wver != 0) { + nodeVersion = wver; + sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion); + } + nodeSStatus = TAOS_SYNC_STATUS_CACHE; sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { @@ -326,11 +351,18 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } void *syncRestoreData(void *param) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) { + sError("failed to restore data, invalid peer rid:%" PRId64, rid); + return NULL; + } + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); + sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); @@ -352,12 +384,15 @@ void *syncRestoreData(void *param) { (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; - sInfo("%s, sync over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); taosClose(pPeer->syncFd); syncCloseRecvBuffer(pNode); __sync_fetch_and_sub(&tsSyncNum, 1); - syncDecPeerRef(pPeer); + + // The ref is obtained in both the create thread and the current thread, so it is released twice + syncReleasePeer(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 36b197dd46484c7ce676391aea5dc4efaf00b9d8..d755ee9aa6f06d27eac144da67295ba02f98610a 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -58,7 +58,7 @@ static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); if (code != 0) { - sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); + sDebug("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); return -1; } @@ -88,11 +88,14 @@ static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo)); - SFileAck fileAck = {0}; + SFileAck fileAck; memset(&fileAck, 0, sizeof(SFileAck)); int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; - if (syncGetFileVersion(pNode, pPeer) < 0) return -1; + if (syncGetFileVersion(pNode, pPeer) < 0) { + pPeer->fileChanged = 1; + return -1; + } while (1) { // retrieve file info @@ -100,12 +103,13 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.size = 0; fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); - // fileInfo.size = htonl(size); - sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); + syncBuildFileInfo(&fileInfo, pNode->vgId); + sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, fileInfo.name, + fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic); // send the file info - int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); - if (ret < 0) { + int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo)); + if (ret != sizeof(SFileInfo)) { code = -1; sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; @@ -119,13 +123,20 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { } // wait for the ack from peer - ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); - if (ret < 0) { + ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck)); + if (ret != sizeof(SFileAck)) { code = -1; sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } + ret = syncCheckHead((SSyncHead*)(&fileAck)); + if (ret != 0) { + code = -1; + sError("%s, failed to check file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(ret)); + break; + } + // set the peer sync version pPeer->sversion = fileInfo.fversion; @@ -134,6 +145,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; sDebug("%s, %s is the same", pPeer->id, fileInfo.name); continue; + } else { + sDebug("%s, %s will be sent", pPeer->id, fileInfo.name); } // get the full path to file @@ -181,7 +194,7 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { } if (ret == 0) { - sTrace("sfd:%d, read to the end of file, ret:%d", sfd, ret); + sDebug("sfd:%d, read to the end of file, ret:%d", sfd, ret); return 0; } @@ -240,7 +253,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi break; } - sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); + sDebug("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); int32_t wsize = code; int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); @@ -384,12 +397,15 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { } if (code == 0) { - pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; - sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - SWalHead walHead; memset(&walHead, 0, sizeof(walHead)); - taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); + if (taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)) == sizeof(walHead)) { + pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; + sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + } else { + sError("%s, failed to send last wal record since %s", pPeer->id, strerror(errno)); + code = -1; + } } else { sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code); } @@ -400,24 +416,22 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFirstPkt firstPkt; - memset(&firstPkt, 0, sizeof(firstPkt)); - firstPkt.syncHead.type = TAOS_SMSG_SYNC_DATA; - firstPkt.syncHead.vgId = pNode->vgId; - tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); - firstPkt.port = tsSyncPort; + SSyncMsg msg; + syncBuildSyncDataMsg(&msg, pNode->vgId); - if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) < 0) { - sError("%s, failed to send sync firstPkt since %s", pPeer->id, strerror(errno)); + if (taosWriteMsg(pPeer->syncFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sError("%s, failed to send sync-data msg since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId); return -1; } + sDebug("%s, send sync-data msg to peer, tranId:%u", pPeer->id, msg.tranId); - SFirstPktRsp firstPktRsp; - if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { - sError("%s, failed to read sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + SSyncRsp rsp; + if (taosReadMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) { + sError("%s, failed to read sync-data rsp since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId); return -1; } + sDebug("%s, recv sync-data rsp from peer, tranId:%u rsp-tranId:%u", pPeer->id, msg.tranId, rsp.tranId); return 0; } @@ -450,9 +464,17 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { } void *syncRetrieveData(void *param) { - SSyncPeer *pPeer = (SSyncPeer *)param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) { + sError("failed to retrieve data, invalid peer rid:%" PRId64, rid); + return NULL; + } + SSyncNode *pNode = pPeer->pSyncNode; + taosBlockSIGPIPE(); + sInfo("%s, start to retrieve data, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); @@ -479,7 +501,12 @@ void *syncRetrieveData(void *param) { pPeer->fileChanged = 0; taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); + + // The ref is obtained in both the create thread and the current thread, so it is released twice + sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + + syncReleasePeer(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/syncTcp.c similarity index 87% rename from src/sync/src/taosTcpPool.c rename to src/sync/src/syncTcp.c index eb05cf7c6f8d8d9a068b60c54f0ae1426e62c429..474466673798ba027518a4bf5c3375492ce9fb60 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/syncTcp.c @@ -19,10 +19,10 @@ #include "tutil.h" #include "tsocket.h" #include "taoserror.h" -#include "taosTcpPool.h" #include "twal.h" #include "tsync.h" #include "syncInt.h" +#include "syncTcp.h" typedef struct SThreadObj { pthread_t thread; @@ -42,17 +42,17 @@ typedef struct SPoolObj { typedef struct { SThreadObj *pThread; - void * ahandle; + int64_t handleId; int32_t fd; int32_t closedByApp; } SConnObj; -static void *taosAcceptPeerTcpConnection(void *argv); -static void *taosProcessTcpData(void *param); -static void taosStopPoolThread(SThreadObj *pThread); -static SThreadObj *taosGetTcpThread(SPoolObj *pPool); +static void *syncAcceptPeerTcpConnection(void *argv); +static void *syncProcessTcpData(void *param); +static void syncStopPoolThread(SThreadObj *pThread); +static SThreadObj *syncGetTcpThread(SPoolObj *pPool); -void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { +void *syncOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_t thattr; SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); @@ -80,7 +80,7 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) { + if (pthread_create(&(pPool->thread), &thattr, (void *)syncAcceptPeerTcpConnection, pPool) != 0) { sError("failed to create accept thread for TCP server since %s", strerror(errno)); close(pPool->acceptFd); tfree(pPool->pThread); @@ -94,7 +94,7 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { return pPool; } -void taosCloseTcpThreadPool(void *param) { +void syncCloseTcpThreadPool(void *param) { SPoolObj * pPool = param; SThreadObj *pThread; @@ -103,7 +103,7 @@ void taosCloseTcpThreadPool(void *param) { for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { pThread = pPool->pThread[i]; - if (pThread) taosStopPoolThread(pThread); + if (pThread) syncStopPoolThread(pThread); } sDebug("%p TCP pool is closed", pPool); @@ -112,7 +112,7 @@ void taosCloseTcpThreadPool(void *param) { tfree(pPool); } -void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -122,7 +122,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { return NULL; } - SThreadObj *pThread = taosGetTcpThread(pPool); + SThreadObj *pThread = syncGetTcpThread(pPool); if (pThread == NULL) { tfree(pConn); return NULL; @@ -130,7 +130,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { pConn->fd = connFd; pConn->pThread = pThread; - pConn->ahandle = pPeer; + pConn->handleId = rid; pConn->closedByApp = 0; event.events = EPOLLIN | EPOLLRDHUP; @@ -149,7 +149,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { return pConn; } -void taosFreeTcpConn(void *param) { +void syncFreeTcpConn(void *param) { SConnObj * pConn = param; SThreadObj *pThread = pConn->pThread; @@ -164,7 +164,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { SPoolInfo * pInfo = &pPool->info; if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR); - (*pInfo->processBrokenLink)(pConn->ahandle); + (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); @@ -175,7 +175,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { #define maxEvents 10 -static void *taosProcessTcpData(void *param) { +static void *syncProcessTcpData(void *param) { SThreadObj *pThread = (SThreadObj *)param; SPoolObj * pPool = pThread->pPool; SPoolInfo * pInfo = &pPool->info; @@ -221,8 +221,8 @@ static void *taosProcessTcpData(void *param) { } if (pConn->closedByApp == 0) { - if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) { - taosFreeTcpConn(pConn); + if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) { + syncFreeTcpConn(pConn); continue; } } @@ -239,7 +239,7 @@ static void *taosProcessTcpData(void *param) { return NULL; } -static void *taosAcceptPeerTcpConnection(void *argv) { +static void *syncAcceptPeerTcpConnection(void *argv) { SPoolObj * pPool = (SPoolObj *)argv; SPoolInfo *pInfo = &pPool->info; @@ -268,7 +268,7 @@ static void *taosAcceptPeerTcpConnection(void *argv) { return NULL; } -static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { +static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { SThreadObj *pThread = pPool->pThread[pPool->nextId]; if (pThread) return pThread; @@ -286,7 +286,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread); + int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)syncProcessTcpData, pThread); pthread_attr_destroy(&thattr); if (ret != 0) { @@ -303,7 +303,7 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { return pThread; } -static void taosStopPoolThread(SThreadObj *pThread) { +static void syncStopPoolThread(SThreadObj *pThread) { pthread_t thread = pThread->thread; if (!taosCheckPthreadValid(thread)) { return; diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 5a64a0a36a9c5e0557e78216ba41a48a23cc874b..161105d86c9a45fb1b8827f01701a43a3e08000c 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -100,7 +100,7 @@ int processRpcMsg(void *item) { pHead->msgType = pMsg->msgType; pHead->len = pMsg->contLen; - uDebug("ver:%" PRIu64 ", pkt from client processed", pHead->version); + uDebug("ver:%" PRIu64 ", rsp from client processed", pHead->version); writeIntoWal(pHead); syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC); @@ -275,7 +275,7 @@ int getWalInfo(int32_t vgId, char *name, int64_t *index) { int writeToCache(int32_t vgId, void *data, int type) { SWalHead *pHead = data; - uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type); + uDebug("rsp from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type); int msgSize = pHead->len + sizeof(SWalHead); void *pMsg = taosAllocateQitem(msgSize); diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5c978abd1da5cc5caa17450661d980ca648763c1..50679749036d7b38e124aad6980cdd44d40998a8 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -31,19 +31,19 @@ extern "C" { #endif -extern int tsdbDebugFlag; +extern int32_t tsdbDebugFlag; -#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} -#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} -#define tsdbWarn(...) { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} -#define tsdbInfo(...) { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} -#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} -#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define TSDB_MAX_TABLE_SCHEMAS 16 -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F -#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF #define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax))) @@ -66,7 +66,8 @@ typedef struct STable { SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void* eventHandler; // TODO void* streamHandler; // TODO - TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure + TSKEY lastKey; + SDataRow lastRow; char* sql; void* cqhandle; SRWLatch latch; // TODO: implementa latch functions @@ -360,8 +361,11 @@ typedef struct { #define TABLE_UID(t) (t)->tableId.uid #define TABLE_TID(t) (t)->tableId.tid #define TABLE_SUID(t) (t)->suid -#define TABLE_LASTKEY(t) (t)->lastKey #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) +#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch)) +#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch)) +#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch)) +#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch)) STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); void tsdbFreeMeta(STsdbMeta* pMeta); @@ -391,7 +395,7 @@ static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, STSchema* pSchema = NULL; STSchema* pTSchema = NULL; - if (lock) taosRLockLatch(&(pDTable->latch)); + if (lock) TSDB_RLOCK_TABLE(pDTable); if (version < 0) { // get the latest version of schema pTSchema = pDTable->schema[pDTable->numOfSchemas - 1]; } else { // get the schema with version @@ -413,7 +417,7 @@ static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, } _exit: - if (lock) taosRUnLockLatch(&(pDTable->latch)); + if (lock) TSDB_RUNLOCK_TABLE(pDTable); return pSchema; } @@ -433,6 +437,11 @@ static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { } } +static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { + ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); + return pTable->lastKey; +} + // ------------------ tsdbBuffer.c #define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 637b02cd32ae8ad8e4077684609dcac23922d8a0..696270d67086286dd3e9d86a69a5cc9cdc4b53eb 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -161,6 +161,11 @@ _err: static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); + SMemTable *pIMem = pRepo->imem; + tsdbLockRepo(pRepo); + pRepo->imem = NULL; + tsdbUnlockRepo(pRepo); + tsdbUnRefMemTable(pRepo, pIMem); sem_post(&(pRepo->readyToCommit)); } @@ -220,7 +225,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe SCommitIter *pIter = iters + tid; if (pIter->pTable == NULL) continue; - taosRLockLatch(&(pIter->pTable->latch)); + TSDB_RLOCK_TABLE(pIter->pTable); if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err; @@ -231,7 +236,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) { - taosRUnLockLatch(&(pIter->pTable->latch)); + TSDB_RUNLOCK_TABLE(pIter->pTable); tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable), tstrerror(terrno)); @@ -239,7 +244,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } } - taosRUnLockLatch(&(pIter->pTable->latch)); + TSDB_RUNLOCK_TABLE(pIter->pTable); // Move the last block to the new .l file if neccessary if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 9d65325001a33dedbf04cf7d065d1d803eeb4e1c..b34b2fa9e6d867527d98036da7972b50c5471c4b 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -77,9 +77,9 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { tsdbDebug( "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d daysPerFile %d keep " - "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d", + "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d update %d cacheLastRow %d", pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, - pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); + pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression, pCfg->update, pCfg->cacheLastRow); return 0; } @@ -281,6 +281,10 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { config.totalBlocks = pCfg->totalBlocks; configChanged = true; } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + config.cacheLastRow = pCfg->cacheLastRow; + configChanged = true; + } if (configChanged) { if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { @@ -475,6 +479,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // update check if (pCfg->update != 0) pCfg->update = 1; + // update cacheLastRow + if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1; + return 0; _err: @@ -692,10 +699,12 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { } } -static int tsdbRestoreInfo(STsdbRepo *pRepo) { +static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pFGroup = NULL; + STsdbCfg * pCfg = &(pRepo->config); + SCompBlock *pBlock = NULL; SFileGroupIter iter; SRWHelper rhelper = {0}; @@ -713,7 +722,32 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err; SCompIdx *pIdx = &(rhelper.curCompIdx); - if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + if (pIdx->offset > 0 && lastKey < pIdx->maxKey) { + pTable->lastKey = pIdx->maxKey; + if (pCfg->cacheLastRow) { // load the block of data + if (tsdbLoadCompInfo(&rhelper, NULL) < 0) goto _err; + + pBlock = rhelper.pCompInfo->blocks + pIdx->numOfBlocks - 1; + if (tsdbLoadBlockData(&rhelper, pBlock, NULL) < 0) goto _err; + + // construct the data row + ASSERT(pTable->lastRow == NULL); + STSchema *pSchema = tsdbGetTableSchema(pTable); + pTable->lastRow = taosTMalloc(schemaTLen(pSchema)); + if (pTable->lastRow == NULL) { + goto _err; + } + + tdInitDataRow(pTable->lastRow, pSchema); + for (int icol = 0; icol < schemaNCols(pSchema); icol++) { + STColumn *pCol = schemaColAt(pSchema, icol); + SDataCol *pDataCol = rhelper.pDataCols[0]->cols + icol; + tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, + pCol->offset); + } + } + } } } @@ -800,6 +834,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->compression); tlen += taosEncodeFixedI8(buf, pCfg->update); + tlen += taosEncodeFixedI8(buf, pCfg->cacheLastRow); return tlen; } @@ -817,6 +852,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->compression)); buf = taosDecodeFixedI8(buf, &(pCfg->update)); + buf = taosDecodeFixedI8(buf, &(pCfg->cacheLastRow)); return buf; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 999e2deb413eb13cc7119ee9042a260366c5f9a3..07f001f68aecba589c68aa078fdb23dd071bbdd5 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -17,6 +17,7 @@ #include "tsdbMain.h" #define TSDB_DATA_SKIPLIST_LEVEL 5 +#define TSDB_MAX_INSERT_BATCH 512 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); static void tsdbFreeMemTable(SMemTable *pMemTable); @@ -35,6 +36,7 @@ static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPB static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter); static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter); +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row); static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, TSKEY now); @@ -205,10 +207,10 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pRepo->mem == NULL) return 0; - SMemTable *pIMem = pRepo->imem; - sem_wait(&(pRepo->readyToCommit)); + ASSERT(pRepo->imem == NULL); + if (pRepo->code != TSDB_CODE_SUCCESS) { tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); } @@ -220,8 +222,6 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { tsdbScheduleCommit(pRepo); if (tsdbUnlockRepo(pRepo) < 0) return -1; - if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1; - return 0; } @@ -606,19 +606,13 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * STable * pTable = NULL; SSubmitBlkIter blkIter = {0}; SDataRow row = NULL; - void ** rows = NULL; + void * rows[TSDB_MAX_INSERT_BATCH] = {0}; int rowCounter = 0; ASSERT(pBlock->tid < pMeta->maxTables); pTable = pMeta->tables[pBlock->tid]; ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); - rows = (void **)calloc(pBlock->numOfRows, sizeof(void *)); - if (rows == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tsdbInitSubmitBlkIter(pBlock, &blkIter); while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) { @@ -632,9 +626,18 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * if (rows[rowCounter] != NULL) { rowCounter++; } + + if (rowCounter == TSDB_MAX_INSERT_BATCH) { + if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + goto _err; + } + + rowCounter = 0; + memset(rows, 0, sizeof(rows)); + } } - if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { goto _err; } @@ -642,11 +645,9 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * pRepo->stat.pointsWritten += points * schemaNCols(pSchema); pRepo->stat.totalStorage += points * schemaVLen(pSchema); - free(rows); return 0; _err: - free(rows); return -1; } @@ -663,9 +664,10 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void return -1; } - if (key > TABLE_LASTKEY(pTable)) { + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + if (key > lastKey) { tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, - REPO_ID(pRepo), key, TABLE_LASTKEY(pTable)); + REPO_ID(pRepo), key, lastKey); return 0; } } @@ -846,8 +848,10 @@ static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **ro if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]); pTableData->numOfRows += dsize; - // TODO: impl delete row thing - if (TABLE_LASTKEY(pTable) < dataRowKey(rows[rowCounter-1])) TABLE_LASTKEY(pTable) = dataRowKey(rows[rowCounter-1]); + // update table latest info + if (tsdbUpdateTableLatestInfo(pRepo, pTable, rows[rowCounter - 1]) < 0) { + return -1; + } return 0; } @@ -889,4 +893,38 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { } } } +} + +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { + STsdbCfg *pCfg = &pRepo->config; + + if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { + if (pCfg->cacheLastRow || pTable->lastRow != NULL) { + SDataRow nrow = pTable->lastRow; + if (taosTSizeof(nrow) < dataRowLen(row)) { + SDataRow orow = nrow; + nrow = taosTMalloc(dataRowLen(row)); + if (nrow == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + dataRowCpy(nrow, row); + TSDB_WLOCK_TABLE(pTable); + pTable->lastKey = dataRowKey(row); + pTable->lastRow = nrow; + TSDB_WUNLOCK_TABLE(pTable); + taosTZfree(orow); + } else { + TSDB_WLOCK_TABLE(pTable); + pTable->lastKey = dataRowKey(row); + dataRowCpy(nrow, row); + TSDB_WUNLOCK_TABLE(pTable); + } + } else { + pTable->lastKey = dataRowKey(row); + } + } + + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 9dfa147c8f35c20cea9b85e38e38a3db60f3572b..7b08178f4949eff0c4e8802f1d04cf1c85554c5b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -377,11 +377,11 @@ int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { // Chage in memory if (pNewSchema != NULL) { // change super table tag schema - taosWLockLatch(&(pTable->pSuper->latch)); + TSDB_WLOCK_TABLE(pTable->pSuper); STSchema *pOldSchema = pTable->pSuper->tagSchema; pTable->pSuper->tagSchema = pNewSchema; tdFreeSchema(pOldSchema); - taosWUnLockLatch(&(pTable->pSuper->latch)); + TSDB_WUNLOCK_TABLE(pTable->pSuper); } bool isChangeIndexCol = (pMsg->colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))); @@ -392,9 +392,9 @@ int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { tsdbWLockRepoMeta(pRepo); tsdbRemoveTableFromIndex(pMeta, pTable); } - taosWLockLatch(&(pTable->latch)); + TSDB_WLOCK_TABLE(pTable); tdSetKVRowDataOfCol(&(pTable->tagVal), pMsg->colId, pMsg->type, POINTER_SHIFT(pMsg->data, pMsg->schemaLen)); - taosWUnLockLatch(&(pTable->latch)); + TSDB_WUNLOCK_TABLE(pTable); if (isChangeIndexCol) { tsdbAddTableIntoIndex(pMeta, pTable, false); tsdbUnlockRepoMeta(pRepo); @@ -587,7 +587,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, STable *pCTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; ASSERT(schemaVersion(pSchema) > schemaVersion(pCTable->schema[pCTable->numOfSchemas - 1])); - taosWLockLatch(&(pCTable->latch)); + TSDB_WLOCK_TABLE(pCTable); if (pCTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) { pCTable->schema[pCTable->numOfSchemas++] = pSchema; } else { @@ -599,7 +599,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); - taosWUnLockLatch(&(pCTable->latch)); + TSDB_WUNLOCK_TABLE(pCTable); if (insertAct) { int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pCTable); @@ -775,6 +775,7 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); + taosTZfree(pTable->lastRow); tfree(pTable->sql); free(pTable); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d5cc566b5541a41e863d4c136746019b4b172ce0..2444283435efeefdccdae2da5a047f23b7330ad5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -110,6 +110,7 @@ typedef struct STsdbQueryHandle { SArray* pTableCheckInfo; // SArray int32_t activeIndex; bool checkFiles; // check file stage + bool cachelastrow; // check if last row cached void* qinfo; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SFileGroup* pFileGroup; @@ -133,7 +134,9 @@ typedef struct STableGroupSupporter { STSchema* pTagSchema; } STableGroupSupporter; -static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); +static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); +static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); +static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); @@ -370,7 +373,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) { - pCond->twindow = changeTableGroupByLastrow(groupList); + pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table if (groupList->numOfTables == 0) { @@ -378,8 +381,14 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab } STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef); + int32_t code = checkForCachedLastRow(pQueryHandle, groupList); + if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 + terrno = code; + return NULL; + } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); + pQueryHandle->type = TSDB_QUERY_TYPE_LAST; return pQueryHandle; } @@ -956,9 +965,9 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } @@ -1704,7 +1713,32 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO return TSDB_CODE_SUCCESS; } -static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) { +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); + +static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + SQueryFilePos* cur = &pQueryHandle->cur; + + while(1) { + int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + + if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + // all data blocks in current file has been checked already, try next file if exists + return getFirstFileDataBlock(pQueryHandle, exists); + } else { // next block of the same file + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; + pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + } + } +} + +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { pQueryHandle->numOfBlocks = 0; SQueryFilePos* cur = &pQueryHandle->cur; @@ -1789,7 +1823,23 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo, exists); + return getDataBlockRv(pQueryHandle, pBlockInfo, exists); +} + +static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { + assert(cur != NULL && numOfBlocks > 0); + return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); +} + +static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + + SQueryFilePos* cur = &pQueryHandle->cur; + assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; } static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { @@ -1800,14 +1850,14 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists if (!pQueryHandle->locateStart) { pQueryHandle->locateStart = true; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); - return getDataBlocksInFilesImpl(pQueryHandle, exists); + return getFirstFileDataBlock(pQueryHandle, exists); } else { // check if current file block is all consumed STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -1815,27 +1865,26 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - // all data blocks in current file has been checked already, try next file if exists - return getDataBlocksInFilesImpl(pQueryHandle, exists); - } else { - // next block of the same file - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->slot += step; - - cur->mixBlock = false; - cur->blockCompleted = false; - - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); - } + // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, pQueryHandle->qinfo); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, + pQueryHandle->qinfo); int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = pQueryHandle->realNumOfRows > 0; + *exists = (pQueryHandle->realNumOfRows > 0); - return code; + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + } + + // current block is empty, try next block in file + // all data blocks in current file has been checked already, try next file if exists + if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { + return getFirstFileDataBlock(pQueryHandle, exists); + } else { + moveToNextDataBlockInCurrentFile(pQueryHandle); + STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pQueryHandle, pNext, exists); } } } @@ -2104,6 +2153,39 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { // restore the pMemRef pQueryHandle->pMemRef = pMemRef; return ret; + } else if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { + // the last row is cached in buffer, return it directly. + // here note that the pQueryHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + SQueryFilePos* cur = &pQueryHandle->cur; + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + + if (++pQueryHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); + if (ret != TSDB_CODE_SUCCESS) { + return false; + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj); + tfree(pRow); + + // update the last key value + pCheckInfo->lastKey = key + step; + + cur->rows = 1; // only one row + cur->lastKey = key + step; + cur->mixBlock = true; + cur->win.skey = key; + cur->win.ekey = key; + + return true; + } + + return false; } if (pQueryHandle->checkFiles) { @@ -2136,7 +2218,57 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return ret; } -STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { +/* + * 1. no data at all (pTable->lastKey = TSKEY_INITIAL_VAL), just return TSKEY_INITIAL_VAL + * 2. has data but not loaded, just return lastKey but not set pRes + * 3. has data and loaded, return lastKey and set pRes + */ +int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { + TSDB_RLOCK_TABLE(pTable); + *lastKey = pTable->lastKey; + + if ((*lastKey) != TSKEY_INITIAL_VAL && pTable->lastRow) { + *pRes = tdDataRowDup(pTable->lastRow); + if (*pRes == NULL) { + TSDB_RUNLOCK_TABLE(pTable); + return TSDB_CODE_TDB_OUT_OF_MEMORY; + } + } + + TSDB_RUNLOCK_TABLE(pTable); + return TSDB_CODE_SUCCESS; +} + +int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { + assert(pQueryHandle != NULL && groupList != NULL); + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + + SArray* group = taosArrayGetP(groupList->pGroupList, 0); + assert(group != NULL); + + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); + + int32_t code = tsdbGetCachedLastRow(pInfo->pTable, &pRow, &key); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->cachelastrow = false; + } else { + pQueryHandle->cachelastrow = (pRow != NULL); + } + + // update the tsdb query time range + if (pQueryHandle->cachelastrow) { + pQueryHandle->window = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = false; + pQueryHandle->activeIndex = -1; // start from -1 + } + + tfree(pRow); + return code; +} + +STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; int32_t totalNumOfTable = 0; @@ -2151,16 +2283,16 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { size_t numOfTables = taosArrayGetSize(pGroup); for(int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); // if the lastKey equals to INT64_MIN, there is no data in this table - TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; - keyInfo.pTable = pKeyInfo->pTable; + keyInfo.pTable = pInfo->pTable; keyInfo.lastKey = key; - pKeyInfo->lastKey = key; + pInfo->lastKey = key; if (key < window.skey) { window.skey = key; @@ -2174,11 +2306,11 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { // clear current group, unref unused table for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); // keyInfo.pTable may be NULL here. - if (pKeyInfo->pTable != keyInfo.pTable) { - tsdbUnRefTable(pKeyInfo->pTable); + if (pInfo->pTable != keyInfo.pTable) { + tsdbUnRefTable(pInfo->pTable); } } @@ -2513,13 +2645,12 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -static bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool tableFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = NULL; - + char* val = NULL; if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { val = (char*) TABLE_NAME(pTable); } else { @@ -2574,15 +2705,17 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) { return true; } +static void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the expression tree SExprTraverseSupp supp = { - .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp, + .nodeFilterFn = (__result_filter_fn_t) tableFilterFp, .setupInfoFn = filterPrepare, .pExtInfo = pSTable->tagSchema, }; - tExprTreeTraverse(pExpr, pSTable->pIndex, pRes, &supp); + getTableListfromSkipList(pExpr, pSTable->pIndex, pRes, &supp); tExprTreeDestroy(&pExpr, destroyHelper); return TSDB_CODE_SUCCESS; } @@ -2824,3 +2957,235 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); pGroupList->numOfTables = 0; } + +static void applyFilterToSkipListNode(SSkipList *pSkipList, tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + // Scan each node in the skiplist by using iterator + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + if (exprTreeApplayFilter(pExpr, pNode, param)) { + taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); + } + } + + tSkipListDestroyIter(iter); +} + +typedef struct { + char* v; + int32_t optr; +} SEndPoint; + +typedef struct { + SEndPoint* start; + SEndPoint* end; +} SQueryCond; + +// todo check for malloc failure +static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { + int32_t optr = queryColInfo->optr; + + if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || + optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { + pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start->optr = queryColInfo->optr; + pCond->start->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + pCond->end = calloc(1, sizeof(SEndPoint)); + pCond->end->optr = queryColInfo->optr; + pCond->end->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { + assert(0); + } + + return TSDB_CODE_SUCCESS; +} + +static void queryIndexedColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = NULL; + + SQueryCond cond = {0}; + if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { + //todo handle error + } + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); + } else { + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); + } + + if (cond.start != NULL) { + int32_t optr = cond.start->optr; + + if (optr == TSDB_RELATION_EQUAL) { // equals + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + if (ret != 0) { + break; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + assert(ret >= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_GREATER) { + continue; + } else { + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; + } + } + } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal + bool comp = true; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + tSkipListDestroyIter(iter); + + comp = true; + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + } else { + assert(0); + } + } else { + int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; + if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); + assert(ret <= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_LESS) { + continue; + } else { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; // no need to compare anymore + } + } + } else { + assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); + if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || + (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } + } + } + + free(cond.start); + free(cond.end); + tSkipListDestroyIter(iter); +} + +static void queryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + + char *pData = SL_GET_NODE_DATA(pNode); + tstr *name = (tstr*) tsdbGetTableName((void*) pData); + + // todo speed up by using hash + if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(name, pQueryInfo->q); + } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = !pQueryInfo->compare(name, pQueryInfo->q); + } + } else { + addToResult = filterFp(pNode, pQueryInfo); + } + + if (addToResult) { + STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(res, &info); + } + } + + tSkipListDestroyIter(iter); +} + +// Apply the filter expression to each node in the skiplist to acquire the qualified nodes in skip list +void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { + if (pExpr == NULL) { + return; + } + + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { + assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); + + param->setupInfoFn(pExpr, param->pExtInfo); + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { + queryIndexedColumn(pSkipList, pQueryInfo, result); + } else { + queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); + } + + return; + } + + // The value of hasPK is always 0. + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); + + //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes + applyFilterToSkipListNode(pSkipList, pExpr, result, param); +} diff --git a/src/util/inc/buildInfo.h b/src/util/inc/buildInfo.h deleted file mode 100644 index 8d169d618d51a1e0a4191c8c8fbe370536970478..0000000000000000000000000000000000000000 --- a/src/util/inc/buildInfo.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef _TS_BUILD_H_ -#define _TS_BUILD_H_ - -extern const char tsVersion[]; -extern const char tsBuildInfo[]; - -#endif diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 5bada93d1c77462fea0d16708fef2546976a8d99..b6b49693f60b15a51eaa35c95555d0f35d36f017 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -32,17 +32,18 @@ typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t keyLen; // length of the key - size_t dataLen; // length of data - int8_t count; // reference count - int8_t removed; // flag to indicate removed + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + int8_t removed; // flag to indicate removed + int8_t count; // reference count char data[]; } SHashNode; #define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) #define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) #define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode)); + typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -115,7 +116,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param dsize * @return */ -void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize); +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize); /** * remove item with the specified key @@ -129,16 +130,14 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); +void taosHashEmpty(SHashObj *pHashObj); + /** * clean up hash table * @param handle */ void taosHashCleanup(SHashObj *pHashObj); -/* -void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *); -*/ - /** * * @param pHashObj diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 665528f14058cdf4edb3c262a285dbbd3482dc26..bc1da9858a3ea66610c6d03364d753b6b2f06313 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -41,6 +41,7 @@ enum { }; enum { + TAOS_CFG_VTYPE_INT8, TAOS_CFG_VTYPE_INT16, TAOS_CFG_VTYPE_INT32, TAOS_CFG_VTYPE_FLOAT, diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index cd5092f30a290c51de49e38b0226bbed637dc0e6..085c10c55198fc92bfacb0628c38f05da508bfb9 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -52,6 +52,8 @@ void *taosIterateRef(int rsetId, int64_t rid); // return the number of references in system int taosListRef(); +#define RID_VALID(x) ((x) > 0) + /* sample code to iterate the refs void demoIterateRefs(int rsetId) { diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index 391cc44accadd785fb46e02aa79cde7617834c70..a339955cc02f31607c63b5c473c57c4d0e6f6f1c 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -33,7 +33,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(SOCKET sockFd); int32_t taosGetFqdn(char *); -uint32_t taosGetIpFromFqdn(const char *); +uint32_t taosGetIpv4FromFqdn(const char *); void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 7aeb2da6b658f4b8fd1cc7e03583a3cc676ea237..e2654f21f4621aa0d8ac34610dd8b887051d75c1 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -31,7 +31,7 @@ extern "C" { typedef struct SStrToken { uint32_t n; uint32_t type; - char * z; + char *z; } SStrToken; /** diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 7a835e87e753d98ec19809e4f3eec32bbe37e55a..1a9c6d314b3fd8a6b3ad6928b6faa4f7fad32757 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -271,10 +271,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetCB(pHashObj, key, keyLen, NULL, NULL, 0); + return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL, 0); } -void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { return NULL; } @@ -313,10 +313,10 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f } if (d != NULL) { - memcpy(d, GET_HASH_NODE_DATA(pNode), dsize); - } else { - data = GET_HASH_NODE_DATA(pNode); + memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); } + + data = GET_HASH_NODE_DATA(pNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -472,38 +472,49 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi return 0; } -void taosHashCleanup(SHashObj *pHashObj) { +void taosHashEmpty(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } + uDebug("hash:%p cleanup hash table", pHashObj); + SHashNode *pNode, *pNext; __wr_lock(&pHashObj->lock, pHashObj->type); - if (pHashObj->hashList) { - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - assert(pEntry->next == 0); - continue; - } + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + assert(pEntry->next == 0); + continue; + } - pNode = pEntry->next; - assert(pNode != NULL); + pNode = pEntry->next; + assert(pNode != NULL); - while (pNode) { - pNext = pNode->next; - FREE_HASH_NODE(pHashObj, pNode); + while (pNode) { + pNext = pNode->next; + FREE_HASH_NODE(pHashObj, pNode); - pNode = pNext; - } + pNode = pNext; } - free(pHashObj->hashList); + pEntry->num = 0; + pEntry->next = NULL; } + pHashObj->size = 0; __wr_unlock(&pHashObj->lock, pHashObj->type); +} + +void taosHashCleanup(SHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + taosHashEmpty(pHashObj); + tfree(pHashObj->hashList); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); @@ -654,7 +665,7 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; - pNewNode->dataLen = dsize; + pNewNode->dataLen = (uint32_t) dsize; pNewNode->count = 1; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2571f11ba41b653e1fcc639cf1c22ef1f50b3448..3afdf41d05cf131922a24c6641d02b95acfe2a93 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -278,7 +278,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen } SCacheDataNode* ptNode = NULL; - taosHashGetCB(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode, sizeof(void*)); + taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode, sizeof(void*)); void* pData = (ptNode != NULL)? ptNode->data:NULL; @@ -547,13 +547,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { return; } + __cache_wr_lock(pCacheObj); STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; - pElem->prev = NULL; + pElem->prev = NULL; + pElem->next = NULL; pNode->inTrashcan = true; pNode->pTNodeHeader = pElem; - __cache_wr_lock(pCacheObj); pElem->next = pCacheObj->pTrash; if (pCacheObj->pTrash) { pCacheObj->pTrash->prev = pElem; @@ -563,8 +564,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->numOfElemsInTrash++; __cache_unlock(pCacheObj); - uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, - pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash); + uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, + pNode->data, pElem, pCacheObj->numOfElemsInTrash); } void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 1d2bd6252f528c938d27aacb41b76b52bf2b334b..0a9f5a98c06c80efc123c6a05f4811b6c66c2615 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -95,6 +95,23 @@ static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { } } +static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { + int32_t value = atoi(input_value); + int8_t *option = (int8_t *)cfg->ptr; + if (value < cfg->minValue || value > cfg->maxValue) { + uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", + cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); + } else { + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { + *option = (int8_t)value; + cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; + } else { + uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, + tsCfgStatusStr[cfg->cfgStatus], *option); + } + } +} + static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { int length = (int)strlen(input_value); char *option = (char *)cfg->ptr; @@ -204,6 +221,9 @@ static void taosReadConfigOption(const char *option, char *value) { if (strcasecmp(cfg->option, option) != 0) continue; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + taosReadInt8Config(cfg, value); + break; case TAOS_CFG_VTYPE_INT16: taosReadInt16Config(cfg, value); break; @@ -376,6 +396,9 @@ void taosPrintGlobalCfg() { blank[blankLen] = 0; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_INT16: uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; @@ -408,6 +431,9 @@ static void taosDumpCfg(SGlobalCfg *cfg) { blank[blankLen] = 0; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_INT16: printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index c0f89e8465c35611cdf6437d8f8a64572ddc2d6b..e0fe51e22a6405273cbb60fbfa04d3a5956105b0 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -64,10 +64,10 @@ typedef struct { } SLogObj; int32_t tsLogKeepDays = 0; -int32_t tsAsyncLog = 1; +int8_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 0.1f; +float tsMinimalLogDirGB = 1.0f; #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; #else @@ -364,7 +364,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { ptm = localtime_r(&curTime, &Tm); len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); @@ -450,7 +450,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . ptm = localtime_r(&curTime, &Tm); len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 89601147a5923ac413800bdb6d134b266f0663b9..e1b834b9499311d6932e5778a0e8116756a11961 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -449,7 +449,7 @@ static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { int32_t endPort = startPort + 11; uInfo("work as client, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); - uint32_t serverIp = taosGetIpFromFqdn(host); + uint32_t serverIp = taosGetIpv4FromFqdn(host); if (serverIp == 0xFFFFFFFF) { uError("failed to resolve fqdn:%s", host); exit(-1); diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index f2db0b331619cac89aae14fe38d9cb0e65f9c034..56f2322a7bd1e0778e2d52667663b3baf5c08c1f 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -249,7 +249,7 @@ void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); va_start(argpointer, format); len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); va_end(argpointer); diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index d72bc5f4121ec0cd71c1c1dad5d9db5421c66a37..7caa1a6c37a873bae2741d2139cee808ae5a73b4 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -61,7 +61,7 @@ taos_queue taosOpenQueue() { pthread_mutex_init(&queue->mutex, NULL); - uTrace("queue:%p is openned", queue); + uTrace("queue:%p is opened", queue); return queue; } @@ -230,7 +230,7 @@ taos_qset taosOpenQset() { pthread_mutex_init(&qset->mutex, NULL); tsem_init(&qset->sem, 0, 0); - uTrace("qset:%p is openned", qset); + uTrace("qset:%p is opened", qset); return qset; } diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 1f83abcb847147da1d9ad0d17d26081176df0c93..3ef45e9b191fddc45220c7b07551b098ae0891ba 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -447,7 +447,7 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { } static void taosLockList(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { @@ -457,7 +457,7 @@ static void taosLockList(int64_t *lockedBy) { } static void taosUnlockList(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { assert(false); } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 20bec169546d6514def2453257346d5f94826187..e3798162e8e949d65c6c6382574a43d94a474ee6 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -280,7 +280,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) { tSkipListRLock(pSkipList); if (iter->order == TSDB_ORDER_ASC) { - if (iter->cur == pSkipList->pTail) return false; + // no data in the skip list + if (iter->cur == pSkipList->pTail || iter->next == NULL) { + iter->cur = pSkipList->pTail; + tSkipListUnlock(pSkipList); + return false; + } + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it @@ -292,9 +298,11 @@ bool tSkipListIterNext(SSkipListIterator *iter) { iter->step++; } else { if (iter->cur == pSkipList->pHead) { + iter->cur = pSkipList->pHead; tSkipListUnlock(pSkipList); return false; } + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 2543c8170896825ae2a94349c0218e7d99ce8592..1a5c3bd64da3274773196da4a03c8c0613810f89 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -18,6 +18,10 @@ #include "tsocket.h" #include "taoserror.h" +#ifndef SIGPIPE + #define SIGPIPE EPIPE +#endif + int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; @@ -40,9 +44,9 @@ int32_t taosGetFqdn(char *fqdn) { return 0; } -uint32_t taosGetIpFromFqdn(const char *fqdn) { +uint32_t taosGetIpv4FromFqdn(const char *fqdn) { struct addrinfo hints = {0}; - hints.ai_family = AF_UNSPEC; + hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; struct addrinfo *result = NULL; @@ -115,6 +119,10 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nwritten; ptr += nwritten; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); @@ -142,6 +150,10 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nread; ptr += nread; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 0222a6d80a3f36bacde9847b1cc326fe22aa389a..015c687b3d941db3f6414d0d3d92564dd14e0fdb 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -19,7 +19,7 @@ #include "ttimer.h" #include "tutil.h" -extern int32_t tscEmbedded; +extern int8_t tscEmbedded; #define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} #define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} @@ -119,7 +119,7 @@ static void timerDecRef(tmr_obj_t* timer) { } static void lockTimerList(timer_list_t* list) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { @@ -129,7 +129,7 @@ static void lockTimerList(timer_list_t* list) { } static void unlockTimerList(timer_list_t* list) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(list->lockedBy), tid, 0) != tid) { assert(false); tmrError("%" PRId64 " trying to unlock a timer list not locked by current thread.", tid); @@ -225,10 +225,11 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { } static bool removeFromWheel(tmr_obj_t* timer) { - if (timer->wheel >= tListLen(wheels)) { + uint8_t wheelIdx = timer->wheel; + if (wheelIdx >= tListLen(wheels)) { return false; } - time_wheel_t* wheel = wheels + timer->wheel; + time_wheel_t* wheel = wheels + wheelIdx; bool removed = false; pthread_mutex_lock(&wheel->mutex); @@ -256,7 +257,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { static void processExpiredTimer(void* handle, void* arg) { tmr_obj_t* timer = (tmr_obj_t*)handle; - timer->executedBy = taosGetPthreadId(); + timer->executedBy = taosGetSelfPthreadId(); uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED); if (state == TIMER_STATE_WAITING) { const char* fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] execution start."; @@ -405,7 +406,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { return false; } - if (timer->executedBy == taosGetPthreadId()) { + if (timer->executedBy == taosGetSelfPthreadId()) { // taosTmrReset is called in the timer callback, should do nothing in this // case to avoid dead lock. note taosTmrReset must be the last statement // of the callback funtion, will be a bug otherwise. diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index b28eb690fec7656e696498fc2dcffef8076fd47c..7adeda590a509df9554c5dc83fa66758ed8ed9d2 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -45,6 +45,9 @@ typedef struct { int8_t accessState; int8_t isFull; int8_t isCommiting; + int8_t dbReplica; + int8_t dropped; + int8_t reserved; uint64_t version; // current version uint64_t cversion; // version while commit start uint64_t fversion; // version on saved data file @@ -56,14 +59,14 @@ typedef struct { int64_t sync; void * events; void * cq; // continuous query - int32_t cfgVersion; + int32_t dbCfgVersion; + int32_t vgCfgVersion; STsdbCfg tsdbCfg; SSyncCfg syncCfg; SWalCfg walCfg; void * qMgmt; char * rootDir; tsem_t sem; - int8_t dropped; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; pthread_mutex_t statusMutex; } SVnodeObj; diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index e0881db00072d6482c1eb557f5b35743902e39b2..0b32f97939a04b695c641a076e863fe37d85a437 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -22,7 +22,8 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db)); - pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion; + pVnode->dbCfgVersion = vnodeMsg->cfg.dbCfgVersion; + pVnode->vgCfgVersion = vnodeMsg->cfg.vgCfgVersion; pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize; pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks; pVnode->tsdbCfg.daysPerFile = vnodeMsg->cfg.daysPerFile; @@ -33,11 +34,13 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow; pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP; - pVnode->syncCfg.replica = vnodeMsg->cfg.replications; + pVnode->syncCfg.replica = vnodeMsg->cfg.vgReplica; pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum; + pVnode->dbReplica = vnodeMsg->cfg.dbReplica; for (int i = 0; i < pVnode->syncCfg.replica; ++i) { SVnodeDesc *node = &vnodeMsg->nodes[i]; @@ -95,12 +98,20 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } tstrncpy(vnodeMsg.db, db->valuestring, sizeof(vnodeMsg.db)); - cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); - if (!cfgVersion || cfgVersion->type != cJSON_Number) { + cJSON *dbCfgVersion = cJSON_GetObjectItem(root, "cfgVersion"); + if (!dbCfgVersion || dbCfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.cfgVersion = cfgVersion->valueint; + vnodeMsg.cfg.dbCfgVersion = dbCfgVersion->valueint; + + cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion"); + if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file); + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; + } cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { @@ -193,12 +204,21 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.wals = (int8_t)wals->valueint; - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { + cJSON *vgReplica = cJSON_GetObjectItem(root, "replica"); + if (!vgReplica || vgReplica->type != cJSON_Number) { vError("vgId:%d, failed to read %s, replica not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.replications = (int8_t)replica->valueint; + vnodeMsg.cfg.vgReplica = (int8_t)vgReplica->valueint; + + cJSON *dbReplica = cJSON_GetObjectItem(root, "dbReplica"); + if (!dbReplica || dbReplica->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file); + vnodeMsg.cfg.dbReplica = vnodeMsg.cfg.vgReplica; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.dbReplica = (int8_t)dbReplica->valueint; + } cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); if (!quorum || quorum->type != cJSON_Number) { @@ -207,6 +227,15 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow"); + if (!cacheLastRow || cacheLastRow->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file); + vnodeMsg.cfg.cacheLastRow = 0; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.cacheLastRow = (int8_t)cacheLastRow->valueint; + } + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file); @@ -214,7 +243,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } int size = cJSON_GetArraySize(nodeInfos); - if (size != vnodeMsg.cfg.replications) { + if (size != vnodeMsg.cfg.vgReplica) { vError("vgId:%d, failed to read %s, nodeInfos size not matched", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } @@ -278,7 +307,8 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"db\": \"%s\",\n", pMsg->db); - len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.cfgVersion); + len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pMsg->cfg.dbCfgVersion); + len += snprintf(content + len, maxLen - len, " \"vgCfgVersion\": %d,\n", pMsg->cfg.vgCfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pMsg->cfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pMsg->cfg.totalBlocks); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pMsg->cfg.daysPerFile); @@ -291,16 +321,18 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pMsg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pMsg->cfg.walLevel); len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pMsg->cfg.fsyncPeriod); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.replications); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.vgReplica); + len += snprintf(content + len, maxLen - len, " \"dbReplica\": %d,\n", pMsg->cfg.dbReplica); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < pMsg->cfg.replications; i++) { + for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) { SVnodeDesc *node = &pMsg->nodes[i]; dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL); len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId); len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp); - if (i < pMsg->cfg.replications - 1) { + if (i < pMsg->cfg.vgReplica - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index d7e33deb15bf31afc2c7e7d7ca1685a30ccb6955..8f44a248ced6157195955fdf79ec713ea6f25072 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -86,6 +86,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.compression = pVnodeCfg->cfg.compression; tsdbCfg.update = pVnodeCfg->cfg.update; + tsdbCfg.cacheLastRow = pVnodeCfg->cfg.cacheLastRow; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); @@ -154,7 +155,7 @@ int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { SVnodeObj *pVnode = vparam; // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS - // cfgVersion can be corrected by status msg + // dbCfgVersion can be corrected by status msg if (!vnodeSetUpdatingStatus(pVnode)) { vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId); return TSDB_CODE_SUCCESS; diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index cf42690d7d0208f50b781e20a4c7c8885f8c5fba..8469ab12c1ef5833e73058c167afb2777ac76ba9 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -89,7 +89,10 @@ static void vnodeIncRef(void *ptNode) { } void *vnodeAcquire(int32_t vgId) { - SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + SVnodeObj **ppVnode = NULL; + if (tsVnodesHash != NULL) { + ppVnode = taosHashGetClone(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + } if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; @@ -134,10 +137,12 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { SVnodeLoad *pLoad = &pStatus->load[pStatus->openVnodes++]; pLoad->vgId = htonl(pVnode->vgId); - pLoad->cfgVersion = htonl(pVnode->cfgVersion); + pLoad->dbCfgVersion = htonl(pVnode->dbCfgVersion); + pLoad->vgCfgVersion = htonl(pVnode->vgCfgVersion); pLoad->totalStorage = htobe64(totalStorage); pLoad->compStorage = htobe64(compStorage); pLoad->pointsWritten = htobe64(pointsWritten); + pLoad->vnodeVersion = htobe64(pVnode->version); pLoad->status = pVnode->status; pLoad->role = pVnode->role; pLoad->replica = pVnode->syncCfg.replica; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 7ad97396c2df2b105056861e5c2916de92c7d8d9..a972ffec1cb5279b191dc1256b85c28e0e5cd9eb 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -65,13 +65,17 @@ static int32_t vnodeCheckRead(SVnodeObj *pVnode) { return TSDB_CODE_APP_NOT_READY; } - if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica, - syncRole[pVnode->role], pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; + if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { + return TSDB_CODE_SUCCESS; + } + + if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) { + return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica, + syncRole[pVnode->role], pVnode->refCount, pVnode); + return TSDB_CODE_APP_NOT_READY; } void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) { @@ -281,7 +285,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { vTrace("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); // In the retrieve blocking model, only 50% CPU will be used in query processing - if (tsHalfCoresForQuery) { + if (tsRetrieveBlockingModel) { qTableQuery(*qhandle); // do execute query qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); } else { @@ -303,8 +307,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; } else { - void *h1 = qGetResultRetrieveMsg(*qhandle); - assert(h1 == NULL); + //void *h1 = qGetResultRetrieveMsg(*qhandle); + + /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */ + //assert(h1 == NULL); + freehandle = qQueryCompleted(*qhandle); } @@ -380,7 +387,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { freeHandle = true; } else { // result is not ready, return immediately // Only effects in the non-blocking model - if (!tsHalfCoresForQuery) { + if (!tsRetrieveBlockingModel) { if (!buildRes) { assert(pRead->rpcHandle != NULL); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 7cf1a9059839e2111808b9458b35fb1ad0573f57..571c7d667adfd285387ab0c11a308dc7472bb0af 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -101,13 +101,19 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara return syncCode; } -static int32_t vnodeCheckWrite(void *vparam) { - SVnodeObj *pVnode = vparam; +static int32_t vnodeCheckWrite(SVnodeObj *pVnode) { if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { vDebug("vgId:%d, no write auth, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); return TSDB_CODE_VND_NO_WRITE_AUTH; } + if (pVnode->dbReplica != pVnode->syncCfg.replica && + pVnode->syncCfg.nodeInfo[pVnode->syncCfg.replica - 1].nodeId == dnodeGetDnodeId()) { + vDebug("vgId:%d, vnode is balancing and will be dropped, dbReplica:%d vgReplica:%d, refCount:%d pVnode:%p", + pVnode->vgId, pVnode->dbReplica, pVnode->syncCfg.replica, pVnode->refCount, pVnode); + return TSDB_CODE_VND_IS_BALANCING; + } + // tsdb may be in reset state if (pVnode->tsdb == NULL) { vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); @@ -209,29 +215,21 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } -int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { - SVnodeObj *pVnode = vparam; - SWalHead * pHead = wparam; - int32_t code = 0; - - if (qtype == TAOS_QTYPE_RPC) { - code = vnodeCheckWrite(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - } - +static SVWriteMsg *vnodeBuildVWriteMsg(SVnodeObj *pVnode, SWalHead *pHead, int32_t qtype, SRpcMsg *pRpcMsg) { if (pHead->len > TSDB_MAX_WAL_SIZE) { vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version); - return TSDB_CODE_WAL_SIZE_LIMIT; + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + return NULL; } int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len; SVWriteMsg *pWrite = taosAllocateQitem(size); if (pWrite == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return NULL; } - if (rparam != NULL) { - SRpcMsg *pRpcMsg = rparam; + if (pRpcMsg != NULL) { pWrite->rpcMsg = *pRpcMsg; } @@ -241,21 +239,48 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar atomic_add_fetch_32(&pVnode->refCount, 1); + return pWrite; +} + +static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) { + SVnodeObj *pVnode = pWrite->pVnode; + + if (pWrite->qtype == TAOS_QTYPE_RPC) { + int32_t code = vnodeCheckWrite(pVnode); + if (code != TSDB_CODE_SUCCESS) { + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + return code; + } + } + int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); if (queued > MAX_QUEUED_MSG_NUM) { - vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued); - taosMsleep(3); + int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3; + if (ms > 100) ms = 100; + vDebug("vgId:%d, too many msg:%d in vwqueue, flow control %dms", pVnode->vgId, queued, ms); + taosMsleep(ms); } - code = vnodePerformFlowCtrl(pWrite); - if (code != 0) return 0; - vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); - taosWriteQitem(pVnode->wqueue, qtype, pWrite); + taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); return TSDB_CODE_SUCCESS; } +int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { + SVWriteMsg *pWrite = vnodeBuildVWriteMsg(vparam, wparam, qtype, rparam); + if (pWrite == NULL) { + assert(terrno != 0); + return terrno; + } + + int32_t code = vnodePerformFlowCtrl(pWrite); + if (code != 0) return 0; + + return vnodeWriteToWQueueImp(pWrite); +} + void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { SVnodeObj *pVnode = vparam; @@ -269,7 +294,7 @@ void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { SVWriteMsg *pWrite = param; SVnodeObj * pVnode = pWrite->pVnode; - int32_t code = TSDB_CODE_VND_SYNCING; + int32_t code = TSDB_CODE_VND_IS_SYNCING; if (pVnode->flowctrlLevel <= 0) code = TSDB_CODE_VND_IS_FLOWCTRL; @@ -285,7 +310,10 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->processedCount); pWrite->processedCount = 0; - taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); + code = vnodeWriteToWQueueImp(pWrite); + if (code != 0) { + dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } } } } @@ -295,7 +323,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; - if (tsFlowCtrl == 0) { + if (tsEnableFlowCtrl == 0) { int32_t ms = pow(2, pVnode->flowctrlLevel + 2); if (ms > 100) ms = 100; vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); @@ -309,4 +337,4 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { pWrite->processedCount); return TSDB_CODE_VND_ACTION_IN_PROGRESS; } -} +} \ No newline at end of file diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 2253ad5c33906714352ecdcd0a916dbcc307a82f..e67127d6e478332889bec3ac565caff62d475b37 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -297,16 +297,14 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } } - if (pHead->len > size - sizeof(SWalHead)) { - size = sizeof(SWalHead) + pHead->len; - buffer = realloc(buffer, size); - if (buffer == NULL) { - wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); - code = TAOS_SYSTEM_ERROR(errno); + if (pHead->len < 0 || pHead->len > size - sizeof(SWalHead)) { + wError("vgId:%d, file:%s, wal head len out of range, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); break; } - - pHead = buffer; } ret = tfRead(tfd, pHead->cont, pHead->len); diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 7a473ed18c958afa8be3c5b94b04d2fd548a56fd..9a52a2ca833e671fe96acf6800f15e44fba292e3 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -76,7 +76,7 @@ int main(int argc, char *argv[]) { taosInitLog("wal.log", 100000, 10); - SWalCfg walCfg; + SWalCfg walCfg = {0}; walCfg.walLevel = level; walCfg.keep = keep; diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index 5e9fcd15cd3c300429ff65d980fbaf5f8ccf2365..09547710c6891c4f08d594b50298b3c52e89a45b 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -1,18 +1,3 @@ - -properties([pipelineTriggers([githubPush()])]) -node { - git url: 'https://github.com/taosdata/TDengine' -} - - -// execute this before anything else, including requesting any time on an agent -if (currentBuild.rawBuild.getCauses().toString().contains('BranchIndexingCause')) { - print "INFO: Build skipped due to trigger being Branch Indexing" - currentBuild.result = 'ABORTED' // optional, gives a better hint to the user that it's been skipped, rather than the default which shows it's successful - return -} - - def pre_test(){ catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' @@ -21,15 +6,14 @@ def pre_test(){ } sh ''' cd ${WKC} - rm -rf * - cd ${WK} git reset --hard - git checkout develop + git checkout $BRANCH_NAME git pull - cd ${WKC} - rm -rf * - mv ${WORKSPACE}/* . + git submodule update cd ${WK} + git reset --hard + git checkout $BRANCH_NAME + git pull export TZ=Asia/Harbin date rm -rf ${WK}/debug @@ -38,13 +22,13 @@ def pre_test(){ cmake .. > /dev/null make > /dev/null make install > /dev/null - cd ${WKC}/tests ''' return 1 } pipeline { agent none environment{ + WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDinternal/community' } @@ -52,13 +36,14 @@ pipeline { stages { stage('Parallel test stage') { parallel { - stage('python p1') { - agent{label 'p1'} + stage('pytest') { + agent{label '184'} steps { pre_test() sh ''' cd ${WKC}/tests - ./test-all.sh p1 + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh pytest date''' } } @@ -66,6 +51,7 @@ pipeline { agent{label 'master'} steps { pre_test() + sh ''' cd ${WKC}/tests ./test-all.sh b1 @@ -74,9 +60,12 @@ pipeline { } stage('test_crash_gen') { - agent{label "b2"} + agent{label "185"} steps { pre_test() + sh ''' + cd ${WKC}/tests/pytest + ''' catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest @@ -89,8 +78,37 @@ pipeline { ./handle_crash_gen_val_log.sh ''' } + sh''' + systemctl start taosd + sleep 10 + ''' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/gotest + bash batchtest.sh + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single -DskipTests >/dev/null + java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + ''' + } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# + dotnet run + ''' + } sh ''' - date + systemctl stop taosd cd ${WKC}/tests ./test-all.sh b2 date @@ -99,42 +117,152 @@ pipeline { } stage('test_valgrind') { - agent{label "b3"} + agent{label "186"} steps { pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest - ./valgrind-test.sh 2>&1 > mem-error-out.log - ./handle_val_log.sh + nohup taosd >/dev/null & + sleep 10 + python3 concurrent_inquiry.py -c 1 + ''' - } + } sh ''' + cd ${WKC}/tests/pytest + ./valgrind-test.sh 2>&1 > mem-error-out.log + ./handle_val_log.sh + date cd ${WKC}/tests ./test-all.sh b3 date''' } } - stage('python p2'){ - agent{label "p2"} - steps{ - pre_test() - sh ''' - date - cd ${WKC}/tests - ./test-all.sh p2 - date - ''' - - } - } - + stage('arm64_build'){ + agent{label 'arm64'} + steps{ + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + } + stage('arm32_build'){ + agent{label 'arm32'} + steps{ + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + + } + } } } } - -} + post { + success { + emailext ( + subject: "SUCCESSFUL: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: ''' + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${PROJECT_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${CAUSE}
    • +
  • 变更概要：${CHANGES}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • +
  • 变更集：${JELLY_SCRIPT}
    • +
    +
+
+ + ''', + to: "yqliu@taosdata.com,pxiao@taosdata.com", + from: "support@taosdata.com" + ) + } + } +} \ No newline at end of file diff --git a/tests/examples/C#/taosdemo/README.md b/tests/examples/C#/taosdemo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..09e1d659b74f22670c6bfe1e6fa85c7a457d6b13 --- /dev/null +++ b/tests/examples/C#/taosdemo/README.md @@ -0,0 +1,37 @@ +install build environment +=== +yum/apt install mono-complete + +build C# version taosdemo +=== +mcs -out:taosdemo *.cs + +run C# version taosdemo +=== +Usage: mono taosdemo.exe [OPTION...] + + -h host, The host to connect to TDengine. Default is localhost. + -p port, The TCP/IP port number to use for the connection. Default is 0. + -u user, The user name to use when connecting to the server. Default is 'root'. + -P password, The password to use when connecting to the server. Default is 'taosdata'. + -d database, Destination database. Default is 'test'. + -a replica, Set the replica parameters of the database, Default 1, min: 1, max: 3. + -m table_prefix, Table prefix name. Default is 't'. + -s sql file, The select sql file. + -M stable, Use super table. + -o outputfile, Direct output to the named file. Default is './output.txt'. + -q query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC. + -b type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'. + -w length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8 + -l num_of_cols_per_record, The number of columns per record. Default is 3. + -T num_of_threads, The number of threads. Default is 10. + -r num_of_records_per_req, The number of records per request. Default is 1000. + -t num_of_tables, The number of tables. Default is 10000. + -n num_of_records_per_table, The number of records per table. Default is 10000. + -c config_directory, Configuration directory. Default is '/etc/taos/'. + -x flag, Insert only flag. + -O order, Insert mode--0: In order, 1: Out of order. Default is in order. + -R rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50. + -D Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database. + -v Print verbose output + -y Skip read key for continous test, default is not skip diff --git a/tests/examples/C#/taosdemo/TDengineDriver.cs b/tests/examples/C#/taosdemo/TDengineDriver.cs new file mode 120000 index 0000000000000000000000000000000000000000..9bee9fb271a995d8ecec6a07fac2861b2a6f200f --- /dev/null +++ b/tests/examples/C#/taosdemo/TDengineDriver.cs @@ -0,0 +1 @@ +../../../../src/connector/C#/TDengineDriver.cs \ No newline at end of file diff --git a/tests/examples/C#/taosdemo/taosdemo.cs b/tests/examples/C#/taosdemo/taosdemo.cs new file mode 100644 index 0000000000000000000000000000000000000000..8e48fa2c8f2d238b7aebf8cba6bf455cb6b890be --- /dev/null +++ b/tests/examples/C#/taosdemo/taosdemo.cs @@ -0,0 +1,750 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +using System; +using System.Text; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Collections; +using System.Threading; +using System.Diagnostics; + +namespace TDengineDriver +{ + class TDengineTest + { + //connect parameters + private string host; + private string configDir; + private string user; + private string password; + private short port = 0; + + //sql parameters + private string dbName = "db"; + private string stableName = "st"; + private string tablePrefix = "t"; + + private bool isInsertOnly = false; + private int queryMode = 1; + + private long recordsPerTable = 10000; + private int recordsPerRequest = 1; + private int colsPerRecord = 3; + private long batchRows = 1000; + private long numOfTables = 10000; + + private IntPtr conn = IntPtr.Zero; + // private long rowsInserted = 0; + private bool useStable = false; + private short methodOfDelete = 0; + private long numOfThreads = 1; + private long rateOfOutorder = 0; + private bool order = true; + private bool skipReadKey = false; + private bool verbose = false; + + + static void PrintHelp(String[] argv) + { + for (int i = 0; i < argv.Length; ++i) + { + if ("--help" == argv[i]) + { + Console.WriteLine("Usage: mono taosdemo.exe [OPTION...]"); + Console.WriteLine(""); + string indent = " "; + Console.Write("{0}{1}", indent, "-h"); + Console.Write("{0}{1}{2}\n", indent, indent, "host, The host to connect to TDengine. Default is localhost."); + Console.Write("{0}{1}", indent, "-p"); + Console.Write("{0}{1}{2}\n", indent, indent, "port, The TCP/IP port number to use for the connection. Default is 0."); + Console.Write("{0}{1}", indent, "-u"); + Console.Write("{0}{1}{2}\n", indent, indent, "user, The user name to use when connecting to the server. Default is 'root'."); + Console.Write("{0}{1}", indent, "-P"); + Console.Write("{0}{1}{2}\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); + Console.Write("{0}{1}", indent, "-d"); + Console.Write("{0}{1}{2}\n", indent, indent, "database, Destination database. Default is 'test'."); + Console.Write("{0}{1}", indent, "-a"); + Console.Write("{0}{1}{2}\n", indent, indent, "replica, Set the replica parameters of the database, Default 1, min: 1, max: 3."); + Console.Write("{0}{1}", indent, "-m"); + Console.Write("{0}{1}{2}\n", indent, indent, "table_prefix, Table prefix name. Default is 't'."); + Console.Write("{0}{1}", indent, "-s"); + Console.Write("{0}{1}{2}\n", indent, indent, "sql file, The select sql file."); + Console.Write("{0}{1}", indent, "-M"); + Console.Write("{0}{1}{2}\n", indent, indent, "stable, Use super table."); + Console.Write("{0}{1}", indent, "-o"); + Console.Write("{0}{1}{2}\n", indent, indent, "outputfile, Direct output to the named file. Default is './output.txt'."); + Console.Write("{0}{1}", indent, "-q"); + Console.Write("{0}{1}{2}\n", indent, indent, "query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + Console.Write("{0}{1}", indent, "-b"); + Console.Write("{0}{1}{2}\n", indent, indent, "type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'."); + Console.Write("{0}{1}", indent, "-w"); + Console.Write("{0}{1}{2}\n", indent, indent, "length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8"); + Console.Write("{0}{1}", indent, "-l"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_cols_per_record, The number of columns per record. Default is 3."); + Console.Write("{0}{1}", indent, "-T"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_threads, The number of threads. Default is 10."); + Console.Write("{0}{1}", indent, "-r"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_records_per_req, The number of records per request. Default is 1000."); + Console.Write("{0}{1}", indent, "-t"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_tables, The number of tables. Default is 10000."); + Console.Write("{0}{1}", indent, "-n"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_records_per_table, The number of records per table. Default is 10000."); + Console.Write("{0}{1}", indent, "-c"); + Console.Write("{0}{1}{2}\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); + Console.Write("{0}{1}", indent, "-x"); + Console.Write("{0}{1}{2}\n", indent, indent, "flag, Insert only flag."); + Console.Write("{0}{1}", indent, "-O"); + Console.Write("{0}{1}{2}\n", indent, indent, "order, Insert mode--0: In order, 1: Out of order. Default is in order."); + Console.Write("{0}{1}", indent, "-R"); + Console.Write("{0}{1}{2}\n", indent, indent, "rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50."); + Console.Write("{0}{1}", indent, "-D"); + Console.Write("{0}{1}{2}\n", indent, indent, "Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database."); + Console.Write("{0}{1}", indent, "-v"); + Console.Write("{0}{1}{2}\n", indent, indent, "Print verbose output"); + Console.Write("{0}{1}", indent, "-y"); + Console.Write("{0}{1}{2}\n", indent, indent, "Skip read key for continous test, default is not skip"); + + System.Environment.Exit(0); + } + } + } + + public void ReadArgument(String[] argv) + { + host = this.GetArgumentAsString(argv, "-h", "127.0.0.1"); + port = (short)this.GetArgumentAsLong(argv, "-p", 0, 65535, 6030); + user = this.GetArgumentAsString(argv, "-u", "root"); + password = this.GetArgumentAsString(argv, "-P", "taosdata"); + dbName = this.GetArgumentAsString(argv, "-d", "db"); + stableName = this.GetArgumentAsString(argv, "-s", "st"); + tablePrefix = this.GetArgumentAsString(argv, "-m", "t"); + isInsertOnly = this.GetArgumentAsFlag(argv, "-x"); + queryMode = (int)this.GetArgumentAsLong(argv, "-q", 0, 1, 0); + numOfTables = this.GetArgumentAsLong(argv, "-t", 1, 1000000000, 10000); + batchRows = this.GetArgumentAsLong(argv, "-r", 1, 10000, 1000); + recordsPerTable = this.GetArgumentAsLong(argv, "-n", 1, 100000000000, 10000); + recordsPerRequest = (int)this.GetArgumentAsLong(argv, "-r", 1, 10000, 1); + colsPerRecord = (int)this.GetArgumentAsLong(argv, "-l", 1, 1024, 3); + configDir = this.GetArgumentAsString(argv, "-c", "C:/TDengine/cfg"); + useStable = this.GetArgumentAsFlag(argv, "-M"); + + methodOfDelete = (short)this.GetArgumentAsLong(argv, "-D", 0, 3, 0); + numOfThreads = (short)this.GetArgumentAsLong(argv, "-T", 1, 10000, 1); + order = this.GetArgumentAsFlag(argv, "-O"); + rateOfOutorder = this.GetArgumentAsLong(argv, "-R", 0, 100, 0); + + skipReadKey = this.GetArgumentAsFlag(argv, "-y"); + verbose = this.GetArgumentAsFlag(argv, "-v"); + + Console.Write("###################################################################\n"); + Console.Write("# Server IP: {0}\n", host); + Console.Write("# User: {0}\n", user); + Console.Write("# Password: {0}\n", password); + Console.Write("# Use super table: {0}\n", useStable); + Console.Write("# Number of Columns per record: {0}\n", colsPerRecord); + Console.Write("# Number of Threads: {0}\n", numOfThreads); + Console.Write("# Number of Tables: {0}\n", numOfTables); + Console.Write("# Number of Data per Table: {0}\n", recordsPerTable); + Console.Write("# Records/Request: {0}\n", recordsPerRequest); + Console.Write("# Database name: {0}\n", dbName); + Console.Write("# Table prefix: {0}\n", tablePrefix); + Console.Write("# Data order: {0}\n", order); + Console.Write("# Data out of order rate: {0}\n", rateOfOutorder); + Console.Write("# Delete method: {0}\n", methodOfDelete); + Console.Write("# Query Mode: {0}\n", queryMode); + Console.Write("# Insert Only: {0}\n", isInsertOnly); + Console.Write("# Verbose output {0}\n", verbose); + Console.Write("# Test time: {0}\n", DateTime.Now.ToString("h:mm:ss tt")); + + Console.Write("###################################################################\n"); + + if (skipReadKey == false) + { + Console.Write("Press any key to continue..\n"); + Console.ReadKey(); + } + } + + public bool GetArgumentAsFlag(String[] argv, String argName) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName == argv[i]) + { + return true; + } + } + return false; + } + + public long GetArgumentAsLong(String[] argv, String argName, int minVal, long maxVal, int defaultValue) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName != argv[i]) + { + continue; + } + if (i < argc - 1) + { + String tmp = argv[i + 1]; + if (tmp[0] == '-') + { + Console.WriteLine("option {0:G} requires an argument", tmp); + ExitProgram(); + } + + long tmpVal = Convert.ToInt64(tmp); + if (tmpVal < minVal || tmpVal > maxVal) + { + Console.WriteLine("option {0:G} should in range [{1:G}, {2:G}]", argName, minVal, maxVal); + ExitProgram(); + } + + return tmpVal; + } + } + + return defaultValue; + } + + public String GetArgumentAsString(String[] argv, String argName, String defaultValue) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName != argv[i]) + { + continue; + } + if (i < argc - 1) + { + String tmp = argv[i + 1]; + if (tmp[0] == '-') + { + Console.WriteLine("option {0:G} requires an argument", tmp); + ExitProgram(); + } + return tmp; + } + } + + return defaultValue; + } + + static void ExitProgram() + { + TDengine.Cleanup(); + System.Environment.Exit(0); + } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + public void InitTDengine() + { + TDengine.Options((int)TDengineInitOption.TDDB_OPTION_CONFIGDIR, this.configDir); + TDengine.Options((int)TDengineInitOption.TDDB_OPTION_SHELL_ACTIVITY_TIMER, "60"); + TDengine.Init(); + DebugPrint("TDengine Initialization finished\n"); + } + + public void ConnectTDengine() + { + string db = ""; + DebugPrintFormat("host:{0} user:{1}, pass:{2}; db:{3}, port:{4}", + this.host, this.user, this.password, db, this.port); + this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); + if (this.conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + ExitProgram(); + } + else + { + DebugPrint("Connect to TDengine success\n"); + } + } + + public void CreateTablesByThreads() + { + Thread[] threadArr = new Thread[numOfThreads]; + + long quotition = numOfTables / numOfThreads; + if (quotition < 1) + { + numOfThreads = numOfTables; + quotition = 1; + } + + long remainder = 0; + if (numOfThreads != 0) + { + remainder = numOfTables % numOfThreads; + } + + long last = 0; + + for (int i = 0; i < numOfThreads; i++) + { + CreateTableThread createTableThread = new CreateTableThread(); + createTableThread.id = i; + createTableThread.verbose = verbose; + createTableThread.dbName = this.dbName; + createTableThread.tablePrefix = this.tablePrefix; + if (useStable) + { + createTableThread.stableName = stableName; + } + createTableThread.conn = conn; + + createTableThread.start = last; + if (i < remainder) + { + createTableThread.end = last + quotition; + } + else + { + createTableThread.end = last + quotition - 1; + } + last = createTableThread.end + 1; + + threadArr[i] = new Thread(createTableThread.ThreadMain); + threadArr[i].Start(); + threadArr[i].Join(); + } + } + + public void dropDatabase() + { + StringBuilder sql = new StringBuilder(); + sql.Append("DROP DATABASE IF EXISTS ").Append(this.dbName); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + + } + + public void CreateDb() + { + StringBuilder sql = new StringBuilder(); + sql.Append("CREATE DATABASE IF NOT EXISTS ").Append(this.dbName); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + TDengine.FreeResult(res); + } + + public void CreateStable() + { + StringBuilder sql = new StringBuilder(); + + sql.Clear(); + sql.Append("CREATE TABLE IF NOT EXISTS "). + Append(this.dbName).Append(".").Append(this.stableName). + Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + TDengine.FreeResult(res); + } + + public void InsertByThreads() + { + Thread[] threadArr = new Thread[numOfThreads]; + + long quotition = numOfTables / numOfThreads; + if (quotition < 1) + { + numOfThreads = numOfTables; + quotition = 1; + } + + long remainder = 0; + if (numOfThreads != 0) + { + remainder = numOfTables % numOfThreads; + } + + long last = 0; + + for (int i = 0; i < numOfThreads; i++) + { + InsertDataThread insertThread = new InsertDataThread(); + insertThread.id = i; + insertThread.recordsPerTable = recordsPerTable; + insertThread.batchRows = batchRows; + insertThread.numOfTables = numOfTables; + insertThread.verbose = verbose; + insertThread.dbName = this.dbName; + insertThread.tablePrefix = this.tablePrefix; + if (useStable) + { + insertThread.stableName = stableName; + } + insertThread.conn = conn; + + insertThread.start = last; + if (i < remainder) + { + insertThread.end = last + quotition; + } + else + { + insertThread.end = last + quotition - 1; + } + last = insertThread.end + 1; + + threadArr[i] = new Thread(insertThread.ThreadMain); + threadArr[i].Start(); + threadArr[i].Join(); + } + } + + public void ExecuteQuery() + { + // System.DateTime start = new System.DateTime(); + long queryRows = 0; + + for (int i = 0; i < 1/*this.numOfTables*/; ++i) + { + String sql = "select * from " + this.dbName + "." + tablePrefix + i; + // Console.WriteLine(sql); + + IntPtr res = TDengine.Query(conn, sql); + if (res == IntPtr.Zero) + { + Console.WriteLine(sql + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + + int fieldCount = TDengine.FieldCount(res); + // Console.WriteLine("field count: " + fieldCount); + + List metas = TDengine.FetchFields(res); + for (int j = 0; j < metas.Count; j++) + { + TDengineMeta meta = (TDengineMeta)metas[j]; + // Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); + } + + IntPtr rowdata; + StringBuilder builder = new StringBuilder(); + while ((rowdata = TDengine.FetchRows(res)) != IntPtr.Zero) + { + queryRows++; + for (int fields = 0; fields < fieldCount; ++fields) + { + TDengineMeta meta = metas[fields]; + int offset = IntPtr.Size * fields; + IntPtr data = Marshal.ReadIntPtr(rowdata, offset); + + builder.Append("---"); + + if (data == IntPtr.Zero) + { + builder.Append("NULL"); + continue; + } + + switch ((TDengineDataType)meta.type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + bool v1 = Marshal.ReadByte(data) == 0 ? false : true; + builder.Append(v1); + break; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + byte v2 = Marshal.ReadByte(data); + builder.Append(v2); + break; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + short v3 = Marshal.ReadInt16(data); + builder.Append(v3); + break; + case TDengineDataType.TSDB_DATA_TYPE_INT: + int v4 = Marshal.ReadInt32(data); + builder.Append(v4); + break; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + long v5 = Marshal.ReadInt64(data); + builder.Append(v5); + break; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); + builder.Append(v6); + break; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); + builder.Append(v7); + break; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + string v8 = Marshal.PtrToStringAnsi(data); + builder.Append(v8); + break; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + long v9 = Marshal.ReadInt64(data); + builder.Append(v9); + break; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + string v10 = Marshal.PtrToStringAnsi(data); + builder.Append(v10); + break; + } + } + builder.Append("---"); + + if (queryRows <= 10) + { + Console.WriteLine(builder.ToString()); + } + builder.Clear(); + } + + if (TDengine.ErrorNo(res) != 0) + { + Console.Write("Query is not complete, Error {0:G}", + TDengine.ErrorNo(res), TDengine.Error(res)); + } + + TDengine.FreeResult(res); + } + /* + System.DateTime end = new System.DateTime(); + TimeSpan ts = end - start; + + Console.Write("Total {0:G} rows inserted, {1:G} rows query, time spend {2:G} seconds.\n" + , this.rowsInserted, queryRows, ts.TotalSeconds); + */ + } + + public void CloseConnection() + { + if (this.conn != IntPtr.Zero) + { + TDengine.Close(this.conn); + } + } + + // Main entry + static void Main(string[] args) + { + PrintHelp(args); + + TDengineTest tester = new TDengineTest(); + tester.ReadArgument(args); + + tester.InitTDengine(); + tester.ConnectTDengine(); + tester.dropDatabase(); + tester.CreateDb(); + + if (tester.useStable == true) + { + tester.CreateStable(); + } + + tester.CreateTablesByThreads(); + + Stopwatch watch = Stopwatch.StartNew(); + tester.InsertByThreads(); + watch.Stop(); + double elapsedMs = watch.Elapsed.TotalMilliseconds; + + Console.WriteLine("Spent {0} seconds to insert {1} records with {2} record(s) per request: {3} records/second", + elapsedMs / 1000, + tester.recordsPerTable * tester.numOfTables, + tester.batchRows, + (tester.recordsPerTable * tester.numOfTables * 1000) / elapsedMs); + + tester.ExecuteQuery(); + tester.CloseConnection(); + + Console.WriteLine("End."); + } + + public class InsertDataThread + { + public long id { set; get; } + public long start { set; get; } + public long end { set; get; } + public string dbName { set; get; } + public IntPtr conn { set; get; } + public string tablePrefix { set; get; } + public string stableName { set; get; } + public long recordsPerTable { set; get; } + public long batchRows { set; get; } + public long numOfTables { set; get; } + public bool verbose { set; get; } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + public void ThreadMain() + { + DebugPrintFormat("InsertDataThread {0} from {1} to {2}", id, start, end); + StringBuilder sql = new StringBuilder(); + long beginTimestamp = 1551369600000L; + long rowsInserted = 0; + + // System.DateTime startTime = new System.DateTime(); + long i = 0; + while (i < recordsPerTable) + { + for (long table = start; table <= end; ++table) + { + long inserted = i; + + sql.Clear(); + sql.Append("INSERT INTO "). + Append(this.dbName).Append(".").Append(this.tablePrefix).Append(table). + Append(" VALUES"); + for (int batch = 0; batch < this.batchRows; ++batch) + { + sql.Append("(") + .Append(beginTimestamp + i + batch) + .Append(", 1, 2, 3,") + .Append(i + batch) + .Append(", 5, 6, 7, 'abc', 'def')"); + + } + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res == IntPtr.Zero) + { + DebugPrint(sql.ToString() + " failure, reason: " + TDengine.Error(res) + "\n"); + } + + inserted += this.batchRows; + + int affectRows = TDengine.AffectRows(res); + rowsInserted += affectRows; + + TDengine.FreeResult(res); + if (table == end) + { + i = inserted; + } + } + } + + } + } + + public class CreateTableThread + { + public long id { set; get; } + public long start { set; get; } + public long end { set; get; } + public string dbName { set; get; } + public IntPtr conn { set; get; } + public string tablePrefix { set; get; } + public string stableName { set; get; } + public bool verbose { set; get; } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + public void ThreadMain() + { + DebugPrintFormat("CreateTable {0} from {1} to {2}", id, start, end); + + StringBuilder sql = new StringBuilder(); + + for (long tableId = start; tableId <= end; tableId++) + { + sql.Clear(); + sql = sql.Append("CREATE TABLE IF NOT EXISTS "). + Append(this.dbName).Append(".").Append(this.tablePrefix).Append(tableId). + Append(" USING ").Append(this.dbName).Append(".").Append(this.stableName). + Append(" TAGS(").Append(tableId).Append(")"); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + DebugPrint(sql.ToString() + " failure, reason: " + TDengine.Error(res) + "\n"); + ExitProgram(); + } + TDengine.FreeResult(res); + } + + } + } + } +} diff --git a/tests/examples/lua/OpenResty/conf/nginx.conf b/tests/examples/lua/OpenResty/conf/nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..2f838c21fccd99ead5641fd7eea1e55b49851fa2 --- /dev/null +++ b/tests/examples/lua/OpenResty/conf/nginx.conf @@ -0,0 +1,21 @@ +worker_processes 1; +user root; +error_log logs/error.log; +events { + worker_connections 1024; +} + +http { + lua_package_path '$prefix/lua/?.lua;$prefix/rest/?.lua;/blah/?.lua;;'; + lua_package_cpath "$prefix/so/?.so;;"; + lua_code_cache off; + server { + listen 7000; + server_name restapi; + charset utf-8; + lua_need_request_body on; + location ~ ^/api/([-_a-zA-Z0-9/]+) { + content_by_lua_file rest/$1.lua; + } + } +} diff --git a/tests/examples/lua/OpenResty/logs/.gitignore b/tests/examples/lua/OpenResty/logs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..ad8530e1c3e3c88fcce3b50abf7cc006333f5522 --- /dev/null +++ b/tests/examples/lua/OpenResty/logs/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/tests/examples/lua/OpenResty/rest/test.lua b/tests/examples/lua/OpenResty/rest/test.lua new file mode 100644 index 0000000000000000000000000000000000000000..179950cbe7cc294cd53a538baecefda28fe30bcc --- /dev/null +++ b/tests/examples/lua/OpenResty/rest/test.lua @@ -0,0 +1,83 @@ +local driver = require "luaconnector51" +local cjson = require "cjson" +ngx.say("start time:"..os.time()) + + +local config = { + host = "127.0.0.1", + port = 6030, + database = "", + user = "root", + password = "taosdata", + max_packet_size = 1024 * 1024 +} + +local conn +local res = driver.connect(config) +if res.code ~=0 then + ngx.say("connect--- failed: "..res.error) + return +else + conn = res.conn + ngx.say("connect--- pass.") +end + +local res = driver.query(conn,"drop database if exists nginx") +if res.code ~=0 then + ngx.say("drop db--- failed: "..res.error) +else + ngx.say("drop db--- pass.") +end +res = driver.query(conn,"create database nginx") +if res.code ~=0 then + ngx.say("create db--- failed: "..res.error) + +else + ngx.say("create db--- pass.") +end + +res = driver.query(conn,"use nginx") +if res.code ~=0 then + ngx.say("select db--- failed: "..res.error) +else + ngx.say("select db--- pass.") +end + +res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") +if res.code ~=0 then + ngx.say("create table---failed: "..res.error) + +else + ngx.say("create table--- pass.") +end + +res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") +if res.code ~=0 then + ngx.say("insert records failed: "..res.error) + return +else + if(res.affected == 3) then + ngx.say("insert records--- pass") + else + ngx.say("insert records---failed: expect 3 affected records, actually affected "..res.affected) + end +end + +res = driver.query(conn,"select * from m1") + +if res.code ~=0 then + ngx.say("select failed: "..res.error) + return +else + ngx.say(cjson.encode(res)) + if (#(res.item) == 3) then + ngx.say("select--- pass") + else + ngx.say("select--- failed: expect 3 affected records, actually received "..#(res.item)) + end + +end +driver.close(conn) +ngx.say("end time:"..os.time()) +--ngx.log(ngx.ERR,"in test file.") + diff --git a/tests/examples/lua/OpenResty/so/luaconnector51.so b/tests/examples/lua/OpenResty/so/luaconnector51.so new file mode 100755 index 0000000000000000000000000000000000000000..442de6e39f909e1aeb869988722b84795c048855 Binary files /dev/null and b/tests/examples/lua/OpenResty/so/luaconnector51.so differ diff --git a/tests/examples/lua/README.md b/tests/examples/lua/README.md index 8efc52b35c96d7a45c2a26c4a6b9e3c2e66db5cc..dd9c9d07874e455329e43c7f77e806eb3634622c 100644 --- a/tests/examples/lua/README.md +++ b/tests/examples/lua/README.md @@ -2,13 +2,13 @@ It's a Lua implementation for [TDengine](https://github.com/taosdata/TDengine), an open-sourced big data platform designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. You may need to install Lua5.3 . -## Dependencies +## Lua Dependencies - Lua: ``` https://www.lua.org/ ``` -## Run with Sample +## Run with Lua Sample Build driver lib: ``` @@ -18,3 +18,26 @@ Run lua sample: ``` lua test.lua ``` + +## OpenResty Dependencies +- OpenResty: +``` +http://openresty.org +``` +## Run with OpenResty Sample +**This section demonstrates how to get binary file for connector. To be convenient for trial, an connector has been put into OpenResty work directory. +Because of difference on C API between Lua5.3 and Lua5.1, the files needed by connector for OpenResty are stored in local source directory and configured in script build.sh.** + +Build driver lib: +``` +cd lua51 +./build.sh +``` +Run OpenResty sample: +``` +cd .. +cd OpenResty +sudo openresty -p . +curl http://127.0.0.1:7000/api/test +``` + diff --git a/tests/examples/lua/lua51/build.sh b/tests/examples/lua/lua51/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..da2981bf7d748a7a42782ad34c0db5b7c666c437 --- /dev/null +++ b/tests/examples/lua/lua51/build.sh @@ -0,0 +1,2 @@ +gcc lua_connector51.c -fPIC -shared -o luaconnector51.so -Wall -ltaos + diff --git a/tests/examples/lua/lua51/lauxlib.h b/tests/examples/lua/lua51/lauxlib.h new file mode 100644 index 0000000000000000000000000000000000000000..a44f0272b3ab0745b0ce798d377b40d2cce113cd --- /dev/null +++ b/tests/examples/lua/lua51/lauxlib.h @@ -0,0 +1,161 @@ +/* +** $Id: lauxlib.h,v 1.88.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions for building Lua libraries +** See Copyright Notice in lua.h +*/ + + +#ifndef lauxlib_h +#define lauxlib_h + + +#include +#include + +#include "lua.h" + + +/* extra error code for `luaL_load' */ +#define LUA_ERRFILE (LUA_ERRERR+1) + +typedef struct luaL_Reg { + const char *name; + lua_CFunction func; +} luaL_Reg; + +LUALIB_API void (luaL_openlib) (lua_State *L, const char *libname, + const luaL_Reg *l, int nup); +LUALIB_API void (luaL_register) (lua_State *L, const char *libname, + const luaL_Reg *l); +LUALIB_API int (luaL_getmetafield) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_callmeta) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_typerror) (lua_State *L, int narg, const char *tname); +LUALIB_API int (luaL_argerror) (lua_State *L, int numarg, const char *extramsg); +LUALIB_API const char *(luaL_checklstring) (lua_State *L, int numArg, + size_t *l); +LUALIB_API const char *(luaL_optlstring) (lua_State *L, int numArg, + const char *def, size_t *l); +LUALIB_API lua_Number (luaL_checknumber) (lua_State *L, int numArg); +LUALIB_API lua_Number (luaL_optnumber) (lua_State *L, int nArg, lua_Number def); + +LUALIB_API lua_Integer (luaL_checkinteger) (lua_State *L, int numArg); +LUALIB_API lua_Integer (luaL_optinteger) (lua_State *L, int nArg, + lua_Integer def); + +LUALIB_API void (luaL_checkstack) (lua_State *L, int sz, const char *msg); +LUALIB_API void (luaL_checktype) (lua_State *L, int narg, int t); +LUALIB_API void (luaL_checkany) (lua_State *L, int narg); + +LUALIB_API int (luaL_newmetatable) (lua_State *L, const char *tname); +LUALIB_API void *(luaL_checkudata) (lua_State *L, int ud, const char *tname); + +LUALIB_API void (luaL_where) (lua_State *L, int lvl); +LUALIB_API int (luaL_error) (lua_State *L, const char *fmt, ...); + +LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def, + const char *const lst[]); + +/* pre-defined references */ +#define LUA_NOREF (-2) +#define LUA_REFNIL (-1) + +LUALIB_API int (luaL_ref) (lua_State *L, int t); +LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref); + +LUALIB_API int (luaL_loadfile) (lua_State *L, const char *filename); +LUALIB_API int (luaL_loadbuffer) (lua_State *L, const char *buff, size_t sz, + const char *name); +LUALIB_API int (luaL_loadstring) (lua_State *L, const char *s); + +LUALIB_API lua_State *(luaL_newstate) (void); + + +LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s, const char *p, + const char *r); + +LUALIB_API const char *(luaL_findtable) (lua_State *L, int idx, + const char *fname, int szhint); + +/* From Lua 5.2. */ +LUALIB_API int luaL_fileresult(lua_State *L, int stat, const char *fname); +LUALIB_API int luaL_execresult(lua_State *L, int stat); +LUALIB_API int (luaL_loadfilex) (lua_State *L, const char *filename, + const char *mode); +LUALIB_API int (luaL_loadbufferx) (lua_State *L, const char *buff, size_t sz, + const char *name, const char *mode); +LUALIB_API void luaL_traceback (lua_State *L, lua_State *L1, const char *msg, + int level); +LUALIB_API void (luaL_setfuncs) (lua_State *L, const luaL_Reg *l, int nup); +LUALIB_API void (luaL_pushmodule) (lua_State *L, const char *modname, + int sizehint); +LUALIB_API void *(luaL_testudata) (lua_State *L, int ud, const char *tname); +LUALIB_API void (luaL_setmetatable) (lua_State *L, const char *tname); + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define luaL_argcheck(L, cond,numarg,extramsg) \ + ((void)((cond) || luaL_argerror(L, (numarg), (extramsg)))) +#define luaL_checkstring(L,n) (luaL_checklstring(L, (n), NULL)) +#define luaL_optstring(L,n,d) (luaL_optlstring(L, (n), (d), NULL)) +#define luaL_checkint(L,n) ((int)luaL_checkinteger(L, (n))) +#define luaL_optint(L,n,d) ((int)luaL_optinteger(L, (n), (d))) +#define luaL_checklong(L,n) ((long)luaL_checkinteger(L, (n))) +#define luaL_optlong(L,n,d) ((long)luaL_optinteger(L, (n), (d))) + +#define luaL_typename(L,i) lua_typename(L, lua_type(L,(i))) + +#define luaL_dofile(L, fn) \ + (luaL_loadfile(L, fn) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_dostring(L, s) \ + (luaL_loadstring(L, s) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_getmetatable(L,n) (lua_getfield(L, LUA_REGISTRYINDEX, (n))) + +#define luaL_opt(L,f,n,d) (lua_isnoneornil(L,(n)) ? (d) : f(L,(n))) + +/* From Lua 5.2. */ +#define luaL_newlibtable(L, l) \ + lua_createtable(L, 0, sizeof(l)/sizeof((l)[0]) - 1) +#define luaL_newlib(L, l) (luaL_newlibtable(L, l), luaL_setfuncs(L, l, 0)) + +/* +** {====================================================== +** Generic Buffer manipulation +** ======================================================= +*/ + + + +typedef struct luaL_Buffer { + char *p; /* current position in buffer */ + int lvl; /* number of strings in the stack (level) */ + lua_State *L; + char buffer[LUAL_BUFFERSIZE]; +} luaL_Buffer; + +#define luaL_addchar(B,c) \ + ((void)((B)->p < ((B)->buffer+LUAL_BUFFERSIZE) || luaL_prepbuffer(B)), \ + (*(B)->p++ = (char)(c))) + +/* compatibility only */ +#define luaL_putchar(B,c) luaL_addchar(B,c) + +#define luaL_addsize(B,n) ((B)->p += (n)) + +LUALIB_API void (luaL_buffinit) (lua_State *L, luaL_Buffer *B); +LUALIB_API char *(luaL_prepbuffer) (luaL_Buffer *B); +LUALIB_API void (luaL_addlstring) (luaL_Buffer *B, const char *s, size_t l); +LUALIB_API void (luaL_addstring) (luaL_Buffer *B, const char *s); +LUALIB_API void (luaL_addvalue) (luaL_Buffer *B); +LUALIB_API void (luaL_pushresult) (luaL_Buffer *B); + + +/* }====================================================== */ + +#endif diff --git a/tests/examples/lua/lua51/lua.h b/tests/examples/lua/lua51/lua.h new file mode 100644 index 0000000000000000000000000000000000000000..9dcafd690655868115ce53dff26599f5845b12c5 --- /dev/null +++ b/tests/examples/lua/lua51/lua.h @@ -0,0 +1,404 @@ +/* +** $Id: lua.h,v 1.218.1.5 2008/08/06 13:30:12 roberto Exp $ +** Lua - An Extensible Extension Language +** Lua.org, PUC-Rio, Brazil (http://www.lua.org) +** See Copyright Notice at the end of this file +*/ + + +#ifndef lua_h +#define lua_h + +#include +#include + + +#include "luaconf.h" + + +#define LUA_VERSION "Lua 5.1" +#define LUA_RELEASE "Lua 5.1.4" +#define LUA_VERSION_NUM 501 +#define LUA_COPYRIGHT "Copyright (C) 1994-2008 Lua.org, PUC-Rio" +#define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo & W. Celes" + + +/* mark for precompiled code (`Lua') */ +#define LUA_SIGNATURE "\033Lua" + +/* option for multiple returns in `lua_pcall' and `lua_call' */ +#define LUA_MULTRET (-1) + + +/* +** pseudo-indices +*/ +#define LUA_REGISTRYINDEX (-10000) +#define LUA_ENVIRONINDEX (-10001) +#define LUA_GLOBALSINDEX (-10002) +#define lua_upvalueindex(i) (LUA_GLOBALSINDEX-(i)) + + +/* thread status */ +#define LUA_OK 0 +#define LUA_YIELD 1 +#define LUA_ERRRUN 2 +#define LUA_ERRSYNTAX 3 +#define LUA_ERRMEM 4 +#define LUA_ERRERR 5 + + +typedef struct lua_State lua_State; + +typedef int (*lua_CFunction) (lua_State *L); + + +/* +** functions that read/write blocks when loading/dumping Lua chunks +*/ +typedef const char * (*lua_Reader) (lua_State *L, void *ud, size_t *sz); + +typedef int (*lua_Writer) (lua_State *L, const void* p, size_t sz, void* ud); + + +/* +** prototype for memory-allocation functions +*/ +typedef void * (*lua_Alloc) (void *ud, void *ptr, size_t osize, size_t nsize); + + +/* +** basic types +*/ +#define LUA_TNONE (-1) + +#define LUA_TNIL 0 +#define LUA_TBOOLEAN 1 +#define LUA_TLIGHTUSERDATA 2 +#define LUA_TNUMBER 3 +#define LUA_TSTRING 4 +#define LUA_TTABLE 5 +#define LUA_TFUNCTION 6 +#define LUA_TUSERDATA 7 +#define LUA_TTHREAD 8 + + + +/* minimum Lua stack available to a C function */ +#define LUA_MINSTACK 20 + + +/* +** generic extra include file +*/ +#if defined(LUA_USER_H) +#include LUA_USER_H +#endif + + +/* type of numbers in Lua */ +typedef LUA_NUMBER lua_Number; + + +/* type for integer functions */ +typedef LUA_INTEGER lua_Integer; + + + +/* +** state manipulation +*/ +LUA_API lua_State *(lua_newstate) (lua_Alloc f, void *ud); +LUA_API void (lua_close) (lua_State *L); +LUA_API lua_State *(lua_newthread) (lua_State *L); + +LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf); + + +/* +** basic stack manipulation +*/ +LUA_API int (lua_gettop) (lua_State *L); +LUA_API void (lua_settop) (lua_State *L, int idx); +LUA_API void (lua_pushvalue) (lua_State *L, int idx); +LUA_API void (lua_remove) (lua_State *L, int idx); +LUA_API void (lua_insert) (lua_State *L, int idx); +LUA_API void (lua_replace) (lua_State *L, int idx); +LUA_API int (lua_checkstack) (lua_State *L, int sz); + +LUA_API void (lua_xmove) (lua_State *from, lua_State *to, int n); + + +/* +** access functions (stack -> C) +*/ + +LUA_API int (lua_isnumber) (lua_State *L, int idx); +LUA_API int (lua_isstring) (lua_State *L, int idx); +LUA_API int (lua_iscfunction) (lua_State *L, int idx); +LUA_API int (lua_isuserdata) (lua_State *L, int idx); +LUA_API int (lua_type) (lua_State *L, int idx); +LUA_API const char *(lua_typename) (lua_State *L, int tp); + +LUA_API int (lua_equal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_rawequal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_lessthan) (lua_State *L, int idx1, int idx2); + +LUA_API lua_Number (lua_tonumber) (lua_State *L, int idx); +LUA_API lua_Integer (lua_tointeger) (lua_State *L, int idx); +LUA_API int (lua_toboolean) (lua_State *L, int idx); +LUA_API const char *(lua_tolstring) (lua_State *L, int idx, size_t *len); +LUA_API size_t (lua_objlen) (lua_State *L, int idx); +LUA_API lua_CFunction (lua_tocfunction) (lua_State *L, int idx); +LUA_API void *(lua_touserdata) (lua_State *L, int idx); +LUA_API lua_State *(lua_tothread) (lua_State *L, int idx); +LUA_API const void *(lua_topointer) (lua_State *L, int idx); + + +/* +** push functions (C -> stack) +*/ +LUA_API void (lua_pushnil) (lua_State *L); +LUA_API void (lua_pushnumber) (lua_State *L, lua_Number n); +LUA_API void (lua_pushinteger) (lua_State *L, lua_Integer n); +LUA_API void (lua_pushlstring) (lua_State *L, const char *s, size_t l); +LUA_API void (lua_pushstring) (lua_State *L, const char *s); +LUA_API const char *(lua_pushvfstring) (lua_State *L, const char *fmt, + va_list argp); +LUA_API const char *(lua_pushfstring) (lua_State *L, const char *fmt, ...); +LUA_API void (lua_pushcclosure) (lua_State *L, lua_CFunction fn, int n); +LUA_API void (lua_pushboolean) (lua_State *L, int b); +LUA_API void (lua_pushlightuserdata) (lua_State *L, void *p); +LUA_API int (lua_pushthread) (lua_State *L); + + +/* +** get functions (Lua -> stack) +*/ +LUA_API void (lua_gettable) (lua_State *L, int idx); +LUA_API void (lua_getfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawget) (lua_State *L, int idx); +LUA_API void (lua_rawgeti) (lua_State *L, int idx, int n); +LUA_API void (lua_createtable) (lua_State *L, int narr, int nrec); +LUA_API void *(lua_newuserdata) (lua_State *L, size_t sz); +LUA_API int (lua_getmetatable) (lua_State *L, int objindex); +LUA_API void (lua_getfenv) (lua_State *L, int idx); + + +/* +** set functions (stack -> Lua) +*/ +LUA_API void (lua_settable) (lua_State *L, int idx); +LUA_API void (lua_setfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawset) (lua_State *L, int idx); +LUA_API void (lua_rawseti) (lua_State *L, int idx, int n); +LUA_API int (lua_setmetatable) (lua_State *L, int objindex); +LUA_API int (lua_setfenv) (lua_State *L, int idx); + + +/* +** `load' and `call' functions (load and run Lua code) +*/ +LUA_API void (lua_call) (lua_State *L, int nargs, int nresults); +LUA_API int (lua_pcall) (lua_State *L, int nargs, int nresults, int errfunc); +LUA_API int (lua_cpcall) (lua_State *L, lua_CFunction func, void *ud); +LUA_API int (lua_load) (lua_State *L, lua_Reader reader, void *dt, + const char *chunkname); + +LUA_API int (lua_dump) (lua_State *L, lua_Writer writer, void *data); + + +/* +** coroutine functions +*/ +LUA_API int (lua_yield) (lua_State *L, int nresults); +LUA_API int (lua_resume) (lua_State *L, int narg); +LUA_API int (lua_status) (lua_State *L); + +/* +** garbage-collection function and options +*/ + +#define LUA_GCSTOP 0 +#define LUA_GCRESTART 1 +#define LUA_GCCOLLECT 2 +#define LUA_GCCOUNT 3 +#define LUA_GCCOUNTB 4 +#define LUA_GCSTEP 5 +#define LUA_GCSETPAUSE 6 +#define LUA_GCSETSTEPMUL 7 +#define LUA_GCISRUNNING 9 + +LUA_API int (lua_gc) (lua_State *L, int what, int data); + + +/* +** miscellaneous functions +*/ + +LUA_API int (lua_error) (lua_State *L); + +LUA_API int (lua_next) (lua_State *L, int idx); + +LUA_API void (lua_concat) (lua_State *L, int n); + +LUA_API lua_Alloc (lua_getallocf) (lua_State *L, void **ud); +LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud); + +LUA_API void lua_setexdata(lua_State *L, void *exdata); +LUA_API void *lua_getexdata(lua_State *L); + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define lua_pop(L,n) lua_settop(L, -(n)-1) + +#define lua_newtable(L) lua_createtable(L, 0, 0) + +#define lua_register(L,n,f) (lua_pushcfunction(L, (f)), lua_setglobal(L, (n))) + +#define lua_pushcfunction(L,f) lua_pushcclosure(L, (f), 0) + +#define lua_strlen(L,i) lua_objlen(L, (i)) + +#define lua_isfunction(L,n) (lua_type(L, (n)) == LUA_TFUNCTION) +#define lua_istable(L,n) (lua_type(L, (n)) == LUA_TTABLE) +#define lua_islightuserdata(L,n) (lua_type(L, (n)) == LUA_TLIGHTUSERDATA) +#define lua_isnil(L,n) (lua_type(L, (n)) == LUA_TNIL) +#define lua_isboolean(L,n) (lua_type(L, (n)) == LUA_TBOOLEAN) +#define lua_isthread(L,n) (lua_type(L, (n)) == LUA_TTHREAD) +#define lua_isnone(L,n) (lua_type(L, (n)) == LUA_TNONE) +#define lua_isnoneornil(L, n) (lua_type(L, (n)) <= 0) + +#define lua_pushliteral(L, s) \ + lua_pushlstring(L, "" s, (sizeof(s)/sizeof(char))-1) + +#define lua_setglobal(L,s) lua_setfield(L, LUA_GLOBALSINDEX, (s)) +#define lua_getglobal(L,s) lua_getfield(L, LUA_GLOBALSINDEX, (s)) + +#define lua_tostring(L,i) lua_tolstring(L, (i), NULL) + + + +/* +** compatibility macros and functions +*/ + +#define lua_open() luaL_newstate() + +#define lua_getregistry(L) lua_pushvalue(L, LUA_REGISTRYINDEX) + +#define lua_getgccount(L) lua_gc(L, LUA_GCCOUNT, 0) + +#define lua_Chunkreader lua_Reader +#define lua_Chunkwriter lua_Writer + + +/* hack */ +LUA_API void lua_setlevel (lua_State *from, lua_State *to); + + +/* +** {====================================================================== +** Debug API +** ======================================================================= +*/ + + +/* +** Event codes +*/ +#define LUA_HOOKCALL 0 +#define LUA_HOOKRET 1 +#define LUA_HOOKLINE 2 +#define LUA_HOOKCOUNT 3 +#define LUA_HOOKTAILRET 4 + + +/* +** Event masks +*/ +#define LUA_MASKCALL (1 << LUA_HOOKCALL) +#define LUA_MASKRET (1 << LUA_HOOKRET) +#define LUA_MASKLINE (1 << LUA_HOOKLINE) +#define LUA_MASKCOUNT (1 << LUA_HOOKCOUNT) + +typedef struct lua_Debug lua_Debug; /* activation record */ + + +/* Functions to be called by the debuger in specific events */ +typedef void (*lua_Hook) (lua_State *L, lua_Debug *ar); + + +LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar); +LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar); +LUA_API const char *lua_getlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_getupvalue (lua_State *L, int funcindex, int n); +LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n); +LUA_API int lua_sethook (lua_State *L, lua_Hook func, int mask, int count); +LUA_API lua_Hook lua_gethook (lua_State *L); +LUA_API int lua_gethookmask (lua_State *L); +LUA_API int lua_gethookcount (lua_State *L); + +/* From Lua 5.2. */ +LUA_API void *lua_upvalueid (lua_State *L, int idx, int n); +LUA_API void lua_upvaluejoin (lua_State *L, int idx1, int n1, int idx2, int n2); +LUA_API int lua_loadx (lua_State *L, lua_Reader reader, void *dt, + const char *chunkname, const char *mode); +LUA_API const lua_Number *lua_version (lua_State *L); +LUA_API void lua_copy (lua_State *L, int fromidx, int toidx); +LUA_API lua_Number lua_tonumberx (lua_State *L, int idx, int *isnum); +LUA_API lua_Integer lua_tointegerx (lua_State *L, int idx, int *isnum); + +/* From Lua 5.3. */ +LUA_API int lua_isyieldable (lua_State *L); + + +struct lua_Debug { + int event; + const char *name; /* (n) */ + const char *namewhat; /* (n) `global', `local', `field', `method' */ + const char *what; /* (S) `Lua', `C', `main', `tail' */ + const char *source; /* (S) */ + int currentline; /* (l) */ + int nups; /* (u) number of upvalues */ + int linedefined; /* (S) */ + int lastlinedefined; /* (S) */ + char short_src[LUA_IDSIZE]; /* (S) */ + /* private part */ + int i_ci; /* active function */ +}; + +/* }====================================================================== */ + + +/****************************************************************************** +* Copyright (C) 1994-2008 Lua.org, PUC-Rio. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +******************************************************************************/ + + +#endif diff --git a/tests/examples/lua/lua51/lua_connector51.c b/tests/examples/lua/lua51/lua_connector51.c new file mode 100644 index 0000000000000000000000000000000000000000..9b932337febb204eada021ececa02bc59cf6d5db --- /dev/null +++ b/tests/examples/lua/lua51/lua_connector51.c @@ -0,0 +1,323 @@ +#include +#include +#include +#include +#include "lua.h" +#include "lauxlib.h" +#include "lualib.h" +#include + +struct cb_param{ + lua_State* state; + int callback; + void * stream; +}; + +static int l_connect(lua_State *L){ + TAOS * taos=NULL; + const char* host; + const char* database; + const char* user; + const char* password; + int port; + + luaL_checktype(L, 1, LUA_TTABLE); + + lua_getfield(L,-1,"host"); + if (lua_isstring(L,-1)){ + host = lua_tostring(L, -1); + // printf("host = %s\n", host); + } + + lua_getfield(L, 1, "port"); + if (lua_isnumber(L,-1)){ + port = lua_tonumber(L, -1); + //printf("port = %d\n", port); + } + + lua_getfield(L, 1, "database"); + if (lua_isstring(L, -1)){ + database = lua_tostring(L, -1); + //printf("database = %s\n", database); + } + + lua_getfield(L, 1, "user"); + if (lua_isstring(L, -1)){ + user = lua_tostring(L, -1); + //printf("user = %s\n", user); + } + + lua_getfield(L, 1, "password"); + if (lua_isstring(L, -1)){ + password = lua_tostring(L, -1); + //printf("password = %s\n", password); + } + + lua_settop(L,0); + + taos_init(); + + lua_newtable(L); + int table_index = lua_gettop(L); + + taos = taos_connect(host, user,password,database, port); + if (taos == NULL) { + printf("failed to connect server, reason:%s\n", taos_errstr(taos)); + + lua_pushinteger(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + lua_pushlightuserdata(L,NULL); + lua_setfield(L, table_index, "conn"); + }else{ + // printf("success to connect server\n"); + lua_pushinteger(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + lua_pushlightuserdata(L,taos); + lua_setfield(L, table_index, "conn"); + } + + return 1; +} + +static int l_query(lua_State *L){ + TAOS *taos= (TAOS*)lua_topointer(L,1); + const char* s = lua_tostring(L, 2); + TAOS_RES *result; + lua_newtable(L); + int table_index = lua_gettop(L); + + // printf("receive command:%s\r\n",s); + result = taos_query(taos, s); + int32_t code = taos_errno(result); + if( code != 0){ + printf("failed, reason:%s\n", taos_errstr(result)); + lua_pushinteger(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + + return 1; + + }else{ + //printf("success to query.\n"); + TAOS_ROW row; + int rows = 0; + int num_fields = taos_field_count(result); + const TAOS_FIELD *fields = taos_fetch_fields(result); + //char temp[256]; + + const int affectRows = taos_affected_rows(result); + // printf(" affect rows:%d\r\n", affectRows); + lua_pushinteger(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushinteger(L, affectRows); + lua_setfield(L, table_index, "affected"); + lua_newtable(L); + + while ((row = taos_fetch_row(result))) { + //printf("row index:%d\n",rows); + rows++; + + lua_pushnumber(L,rows); + lua_newtable(L); + + for (int i = 0; i < num_fields; ++i) { + if (row[i] == NULL) { + continue; + } + + lua_pushstring(L,fields[i].name); + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + lua_pushinteger(L,*((char *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + lua_pushinteger(L,*((short *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + lua_pushinteger(L,*((int *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: + lua_pushnumber(L,*((float *)row[i])); + break; + case TSDB_DATA_TYPE_DOUBLE: + lua_pushnumber(L,*((double *)row[i])); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + lua_pushstring(L,(char *)row[i]); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_BOOL: + lua_pushinteger(L,*((char *)row[i])); + break; + default: + lua_pushnil(L); + break; + } + + lua_settable(L,-3); + } + + lua_settable(L,-3); + } + taos_free_result(result); + } + + lua_setfield(L, table_index, "item"); + return 1; +} + +void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ + struct cb_param* p = (struct cb_param*) param; + TAOS_FIELD *fields = taos_fetch_fields(result); + int numFields = taos_num_fields(result); + + // printf("\nnumfields:%d\n", numFields); + //printf("\n\r-----------------------------------------------------------------------------------\n"); + + lua_State *L = p->state; + lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); + + lua_newtable(L); + + for (int i = 0; i < numFields; ++i) { + if (row[i] == NULL) { + continue; + } + + lua_pushstring(L,fields[i].name); + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + lua_pushinteger(L,*((char *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + lua_pushinteger(L,*((short *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + lua_pushinteger(L,*((int *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: + lua_pushnumber(L,*((float *)row[i])); + break; + case TSDB_DATA_TYPE_DOUBLE: + lua_pushnumber(L,*((double *)row[i])); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + lua_pushstring(L,(char *)row[i]); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + lua_pushinteger(L,*((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_BOOL: + lua_pushinteger(L,*((char *)row[i])); + break; + default: + lua_pushnil(L); + break; + } + + lua_settable(L, -3); + } + + lua_call(L, 1, 0); + + // printf("-----------------------------------------------------------------------------------\n\r"); +} + +static int l_open_stream(lua_State *L){ + int r = luaL_ref(L, LUA_REGISTRYINDEX); + TAOS * taos = (TAOS*)lua_topointer(L,1); + const char * sqlstr = lua_tostring(L,2); + int stime = luaL_checknumber(L,3); + + lua_newtable(L); + int table_index = lua_gettop(L); + + struct cb_param *p = malloc(sizeof(struct cb_param)); + p->state = L; + p->callback=r; + // printf("r:%d, L:%d\n",r,L); + void * s = taos_open_stream(taos,sqlstr,stream_cb,stime,p,NULL); + if (s == NULL) { + printf("failed to open stream, reason:%s\n", taos_errstr(taos)); + free(p); + lua_pushnumber(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + lua_pushlightuserdata(L,NULL); + lua_setfield(L, table_index, "stream"); + }else{ + // printf("success to open stream\n"); + lua_pushnumber(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, taos_errstr(taos)); + lua_setfield(L, table_index, "error"); + p->stream = s; + lua_pushlightuserdata(L,p); + lua_setfield(L, table_index, "stream");//stream has different content in lua and c. + } + + return 1; +} + +static int l_close_stream(lua_State *L){ + //TODO:get stream and free cb_param + struct cb_param *p = lua_touserdata(L,1); + taos_close_stream(p->stream); + free(p); + return 0; +} + +static int l_close(lua_State *L){ + TAOS *taos= (TAOS*)lua_topointer(L,1); + lua_newtable(L); + int table_index = lua_gettop(L); + + if(taos == NULL){ + lua_pushnumber(L, -1); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, "null pointer."); + lua_setfield(L, table_index, "error"); + }else{ + taos_close(taos); + lua_pushnumber(L, 0); + lua_setfield(L, table_index, "code"); + lua_pushstring(L, "done."); + lua_setfield(L, table_index, "error"); + } + return 1; +} + +static const struct luaL_Reg lib[] = { + {"connect", l_connect}, + {"query", l_query}, + {"close", l_close}, + {"open_stream", l_open_stream}, + {"close_stream", l_close_stream}, + {NULL, NULL} +}; + +extern int luaopen_luaconnector51(lua_State* L) +{ + // luaL_register(L, "luaconnector51", lib); + lua_newtable (L); + luaL_setfuncs(L,lib,0); + return 1; +} diff --git a/tests/examples/lua/lua51/luaconf.h b/tests/examples/lua/lua51/luaconf.h new file mode 100644 index 0000000000000000000000000000000000000000..c72893fd152e23c2a413c2da1d31be7b860b2a39 --- /dev/null +++ b/tests/examples/lua/lua51/luaconf.h @@ -0,0 +1,152 @@ +/* +** Configuration header. +** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h +*/ + +#ifndef luaconf_h +#define luaconf_h + +#ifndef WINVER +#define WINVER 0x0501 +#endif +#include +#include + +/* Default path for loading Lua and C modules with require(). */ +#if defined(_WIN32) +/* +** In Windows, any exclamation mark ('!') in the path is replaced by the +** path of the directory of the executable file of the current process. +*/ +#define LUA_LDIR "!\\lua\\" +#define LUA_CDIR "!\\" +#define LUA_PATH_DEFAULT \ + ".\\?.lua;" "!\\lualib\\?.lua;" LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" +#define LUA_CPATH_DEFAULT \ + ".\\?.dll;" "!\\lualib\\?.so;" LUA_CDIR"?.dll;" LUA_CDIR"loadall.dll" +#else +/* +** Note to distribution maintainers: do NOT patch the following lines! +** Please read ../doc/install.html#distro and pass PREFIX=/usr instead. +*/ +#ifndef LUA_MULTILIB +#define LUA_MULTILIB "lib" +#endif +#ifndef LUA_LMULTILIB +#define LUA_LMULTILIB "lib" +#endif +#define LUA_LROOT "/usr/local" +#define LUA_LUADIR "/lua/5.1/" +#define LUA_LJDIR "/luajit-2.1.0-beta3/" + +#ifdef LUA_ROOT +#define LUA_JROOT LUA_ROOT +#define LUA_RLDIR LUA_ROOT "/share" LUA_LUADIR +#define LUA_RCDIR LUA_ROOT "/" LUA_MULTILIB LUA_LUADIR +#define LUA_RLPATH ";" LUA_RLDIR "?.lua;" LUA_RLDIR "?/init.lua" +#define LUA_RCPATH ";" LUA_RCDIR "?.so" +#else +#define LUA_JROOT LUA_LROOT +#define LUA_RLPATH +#define LUA_RCPATH +#endif + +#define LUA_JPATH ";" LUA_JROOT "/share" LUA_LJDIR "?.lua" +#define LUA_LLDIR LUA_LROOT "/share" LUA_LUADIR +#define LUA_LCDIR LUA_LROOT "/" LUA_LMULTILIB LUA_LUADIR +#define LUA_LLPATH ";" LUA_LLDIR "?.lua;" LUA_LLDIR "?/init.lua" +#define LUA_LCPATH1 ";" LUA_LCDIR "?.so" +#define LUA_LCPATH2 ";" LUA_LCDIR "loadall.so" + +#define LUA_PATH_DEFAULT "./?.lua" LUA_JPATH LUA_LLPATH LUA_RLPATH +#define LUA_CPATH_DEFAULT "./?.so" LUA_LCPATH1 LUA_RCPATH LUA_LCPATH2 +#endif + +/* Environment variable names for path overrides and initialization code. */ +#define LUA_PATH "LUA_PATH" +#define LUA_CPATH "LUA_CPATH" +#define LUA_INIT "LUA_INIT" + +/* Special file system characters. */ +#if defined(_WIN32) +#define LUA_DIRSEP "\\" +#else +#define LUA_DIRSEP "/" +#endif +#define LUA_PATHSEP ";" +#define LUA_PATH_MARK "?" +#define LUA_EXECDIR "!" +#define LUA_IGMARK "-" +#define LUA_PATH_CONFIG \ + LUA_DIRSEP "\n" LUA_PATHSEP "\n" LUA_PATH_MARK "\n" \ + LUA_EXECDIR "\n" LUA_IGMARK "\n" + +/* Quoting in error messages. */ +#define LUA_QL(x) "'" x "'" +#define LUA_QS LUA_QL("%s") + +/* Various tunables. */ +#define LUAI_MAXSTACK 65500 /* Max. # of stack slots for a thread (<64K). */ +#define LUAI_MAXCSTACK 8000 /* Max. # of stack slots for a C func (<10K). */ +#define LUAI_GCPAUSE 200 /* Pause GC until memory is at 200%. */ +#define LUAI_GCMUL 200 /* Run GC at 200% of allocation speed. */ +#define LUA_MAXCAPTURES 32 /* Max. pattern captures. */ + +/* Configuration for the frontend (the luajit executable). */ +#if defined(luajit_c) +#define LUA_PROGNAME "luajit" /* Fallback frontend name. */ +#define LUA_PROMPT "> " /* Interactive prompt. */ +#define LUA_PROMPT2 ">> " /* Continuation prompt. */ +#define LUA_MAXINPUT 512 /* Max. input line length. */ +#endif + +/* Note: changing the following defines breaks the Lua 5.1 ABI. */ +#define LUA_INTEGER ptrdiff_t +#define LUA_IDSIZE 60 /* Size of lua_Debug.short_src. */ +/* +** Size of lauxlib and io.* on-stack buffers. Weird workaround to avoid using +** unreasonable amounts of stack space, but still retain ABI compatibility. +** Blame Lua for depending on BUFSIZ in the ABI, blame **** for wrecking it. +*/ +#define LUAL_BUFFERSIZE (BUFSIZ > 16384 ? 8192 : BUFSIZ) + +/* The following defines are here only for compatibility with luaconf.h +** from the standard Lua distribution. They must not be changed for LuaJIT. +*/ +#define LUA_NUMBER_DOUBLE +#define LUA_NUMBER double +#define LUAI_UACNUMBER double +#define LUA_NUMBER_SCAN "%lf" +#define LUA_NUMBER_FMT "%.14g" +#define lua_number2str(s, n) sprintf((s), LUA_NUMBER_FMT, (n)) +#define LUAI_MAXNUMBER2STR 32 +#define LUA_INTFRMLEN "l" +#define LUA_INTFRM_T long + +/* Linkage of public API functions. */ +#if defined(LUA_BUILD_AS_DLL) +#if defined(LUA_CORE) || defined(LUA_LIB) +#define LUA_API __declspec(dllexport) +#else +#define LUA_API __declspec(dllimport) +#endif +#else +#define LUA_API extern +#endif + +#define LUALIB_API LUA_API + +/* Support for internal assertions. */ +#if defined(LUA_USE_ASSERT) || defined(LUA_USE_APICHECK) +#include +#endif +#ifdef LUA_USE_ASSERT +#define lua_assert(x) assert(x) +#endif +#ifdef LUA_USE_APICHECK +#define luai_apicheck(L, o) { (void)L; assert(o); } +#else +#define luai_apicheck(L, o) { (void)L; } +#endif + +#endif diff --git a/tests/examples/lua/lua51/luajit.h b/tests/examples/lua/lua51/luajit.h new file mode 100644 index 0000000000000000000000000000000000000000..ae14c4ffebdff746b22b2274abf33b5c9d601b26 --- /dev/null +++ b/tests/examples/lua/lua51/luajit.h @@ -0,0 +1,81 @@ +/* +** LuaJIT -- a Just-In-Time Compiler for Lua. http://luajit.org/ +** +** Copyright (C) 2005-2017 Mike Pall. All rights reserved. +** +** Permission is hereby granted, free of charge, to any person obtaining +** a copy of this software and associated documentation files (the +** "Software"), to deal in the Software without restriction, including +** without limitation the rights to use, copy, modify, merge, publish, +** distribute, sublicense, and/or sell copies of the Software, and to +** permit persons to whom the Software is furnished to do so, subject to +** the following conditions: +** +** The above copyright notice and this permission notice shall be +** included in all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +** SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +** +** [ MIT license: http://www.opensource.org/licenses/mit-license.php ] +*/ + +#ifndef _LUAJIT_H +#define _LUAJIT_H + +#include "lua.h" + +#define OPENRESTY_LUAJIT + +#define LUAJIT_VERSION "LuaJIT 2.1.0-beta3" +#define LUAJIT_VERSION_NUM 20100 /* Version 2.1.0 = 02.01.00. */ +#define LUAJIT_VERSION_SYM luaJIT_version_2_1_0_beta3 +#define LUAJIT_COPYRIGHT "Copyright (C) 2005-2017 Mike Pall" +#define LUAJIT_URL "http://luajit.org/" + +/* Modes for luaJIT_setmode. */ +#define LUAJIT_MODE_MASK 0x00ff + +enum { + LUAJIT_MODE_ENGINE, /* Set mode for whole JIT engine. */ + LUAJIT_MODE_DEBUG, /* Set debug mode (idx = level). */ + + LUAJIT_MODE_FUNC, /* Change mode for a function. */ + LUAJIT_MODE_ALLFUNC, /* Recurse into subroutine protos. */ + LUAJIT_MODE_ALLSUBFUNC, /* Change only the subroutines. */ + + LUAJIT_MODE_TRACE, /* Flush a compiled trace. */ + + LUAJIT_MODE_WRAPCFUNC = 0x10, /* Set wrapper mode for C function calls. */ + + LUAJIT_MODE_MAX +}; + +/* Flags or'ed in to the mode. */ +#define LUAJIT_MODE_OFF 0x0000 /* Turn feature off. */ +#define LUAJIT_MODE_ON 0x0100 /* Turn feature on. */ +#define LUAJIT_MODE_FLUSH 0x0200 /* Flush JIT-compiled code. */ + +/* LuaJIT public C API. */ + +/* Control the JIT engine. */ +LUA_API int luaJIT_setmode(lua_State *L, int idx, int mode); + +/* Low-overhead profiling API. */ +typedef void (*luaJIT_profile_callback)(void *data, lua_State *L, + int samples, int vmstate); +LUA_API void luaJIT_profile_start(lua_State *L, const char *mode, + luaJIT_profile_callback cb, void *data); +LUA_API void luaJIT_profile_stop(lua_State *L); +LUA_API const char *luaJIT_profile_dumpstack(lua_State *L, const char *fmt, + int depth, size_t *len); + +/* Enforce (dynamic) linker error for version mismatches. Call from main. */ +LUA_API void LUAJIT_VERSION_SYM(void); + +#endif diff --git a/tests/examples/lua/lua51/lualib.h b/tests/examples/lua/lua51/lualib.h new file mode 100644 index 0000000000000000000000000000000000000000..6aceabe59218d1863e6493bb77b9bed18bc18bdc --- /dev/null +++ b/tests/examples/lua/lua51/lualib.h @@ -0,0 +1,44 @@ +/* +** Standard library header. +** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h +*/ + +#ifndef _LUALIB_H +#define _LUALIB_H + +#include "lua.h" + +#define LUA_FILEHANDLE "FILE*" + +#define LUA_COLIBNAME "coroutine" +#define LUA_MATHLIBNAME "math" +#define LUA_STRLIBNAME "string" +#define LUA_TABLIBNAME "table" +#define LUA_IOLIBNAME "io" +#define LUA_OSLIBNAME "os" +#define LUA_LOADLIBNAME "package" +#define LUA_DBLIBNAME "debug" +#define LUA_BITLIBNAME "bit" +#define LUA_JITLIBNAME "jit" +#define LUA_FFILIBNAME "ffi" +#define LUA_THRLIBNAME "thread" + +LUALIB_API int luaopen_base(lua_State *L); +LUALIB_API int luaopen_math(lua_State *L); +LUALIB_API int luaopen_string(lua_State *L); +LUALIB_API int luaopen_table(lua_State *L); +LUALIB_API int luaopen_io(lua_State *L); +LUALIB_API int luaopen_os(lua_State *L); +LUALIB_API int luaopen_package(lua_State *L); +LUALIB_API int luaopen_debug(lua_State *L); +LUALIB_API int luaopen_bit(lua_State *L); +LUALIB_API int luaopen_jit(lua_State *L); +LUALIB_API int luaopen_ffi(lua_State *L); + +LUALIB_API void luaL_openlibs(lua_State *L); + +#ifndef lua_assert +#define lua_assert(x) ((void)0) +#endif + +#endif diff --git a/tests/examples/lua/lua_connector.c b/tests/examples/lua/lua_connector.c index 143f16a799244d4b77f3d01c4d2189fa6519b0d9..920d2cdc35c51c833a4d89448ec7e643f555dbc2 100644 --- a/tests/examples/lua/lua_connector.c +++ b/tests/examples/lua/lua_connector.c @@ -13,17 +13,49 @@ struct cb_param{ void * stream; }; +static int l_connect(lua_State *L){ + TAOS * taos=NULL; + const char* host; + const char* database; + const char* user; + const char* password; + int port; + + luaL_checktype(L, 1, LUA_TTABLE); + + lua_getfield(L,-1,"host"); + if (lua_isstring(L,-1)){ + host = lua_tostring(L, -1); + // printf("host = %s\n", host); + } + + lua_getfield(L, 1, "port"); + if (lua_isinteger(L,-1)){ + port = lua_tointeger(L, -1); + //printf("port = %d\n", port); + } + + lua_getfield(L, 1, "database"); + if (lua_isstring(L, -1)){ + database = lua_tostring(L, -1); + //printf("database = %s\n", database); + } + + lua_getfield(L, 1, "user"); + if (lua_isstring(L, -1)){ + user = lua_tostring(L, -1); + //printf("user = %s\n", user); + } + + lua_getfield(L, 1, "password"); + if (lua_isstring(L, -1)){ + password = lua_tostring(L, -1); + //printf("password = %s\n", password); + } + lua_settop(L,0); -static int l_connect(lua_State *L){ - TAOS * taos; - char *host = lua_tostring(L, 1); - char *user = lua_tostring(L, 2); - char *password = lua_tostring(L, 3); - char *database = lua_tostring(L, 4); - int port =luaL_checknumber(L, 5); taos_init(); - lua_newtable(L); int table_index = lua_gettop(L); @@ -31,38 +63,38 @@ static int l_connect(lua_State *L){ if (taos == NULL) { printf("failed to connect server, reason:%s\n", taos_errstr(taos)); - lua_pushnumber(L, -1); + lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); lua_setfield(L, table_index, "error"); lua_pushlightuserdata(L,NULL); lua_setfield(L, table_index, "conn"); }else{ - printf("success to connect server\n"); - lua_pushnumber(L, 0); + // printf("success to connect server\n"); + lua_pushinteger(L, 0); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); lua_setfield(L, table_index, "error"); lua_pushlightuserdata(L,taos); lua_setfield(L, table_index, "conn"); } - + return 1; } static int l_query(lua_State *L){ - TAOS * taos= lua_topointer(L,1); - char *s = lua_tostring(L, 2); + TAOS *taos= (TAOS*)lua_topointer(L,1); + const char* s = lua_tostring(L, 2); TAOS_RES *result; lua_newtable(L); int table_index = lua_gettop(L); // printf("receive command:%s\r\n",s); - result = taos_query(taos,s); - int32_t code = taos_errno(result); + result = taos_query(taos, s); + int32_t code = taos_errno(result); if( code != 0){ printf("failed, reason:%s\n", taos_errstr(result)); - lua_pushnumber(L, -1); + lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); lua_setfield(L, table_index, "error"); @@ -74,12 +106,12 @@ static int l_query(lua_State *L){ TAOS_ROW row; int rows = 0; int num_fields = taos_field_count(result); - TAOS_FIELD *fields = taos_fetch_fields(result); - char temp[256]; + const TAOS_FIELD *fields = taos_fetch_fields(result); + //char temp[256]; - int affectRows = taos_affected_rows(result); + const int affectRows = taos_affected_rows(result); // printf(" affect rows:%d\r\n", affectRows); - lua_pushnumber(L, 0); + lua_pushinteger(L, 0); lua_setfield(L, table_index, "code"); lua_pushinteger(L, affectRows); lua_setfield(L, table_index, "affected"); @@ -150,8 +182,8 @@ void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ TAOS_FIELD *fields = taos_fetch_fields(result); int numFields = taos_num_fields(result); - printf("\nnumfields:%d\n", numFields); - printf("\n\r-----------------------------------------------------------------------------------\n"); + // printf("\nnumfields:%d\n", numFields); + //printf("\n\r-----------------------------------------------------------------------------------\n"); lua_State *L = p->state; lua_rawgeti(L, LUA_REGISTRYINDEX, p->callback); @@ -204,13 +236,13 @@ void stream_cb(void *param, TAOS_RES *result, TAOS_ROW row){ lua_call(L, 1, 0); - printf("-----------------------------------------------------------------------------------\n\r"); + // printf("-----------------------------------------------------------------------------------\n\r"); } static int l_open_stream(lua_State *L){ int r = luaL_ref(L, LUA_REGISTRYINDEX); - TAOS * taos = lua_topointer(L,1); - char * sqlstr = lua_tostring(L,2); + TAOS * taos = (TAOS*)lua_topointer(L,1); + const char * sqlstr = lua_tostring(L,2); int stime = luaL_checknumber(L,3); lua_newtable(L); @@ -253,7 +285,7 @@ static int l_close_stream(lua_State *L){ } static int l_close(lua_State *L){ - TAOS * taos= lua_topointer(L,1); + TAOS *taos= (TAOS*)lua_topointer(L,1); lua_newtable(L); int table_index = lua_gettop(L); @@ -263,7 +295,7 @@ static int l_close(lua_State *L){ lua_pushstring(L, "null pointer."); lua_setfield(L, table_index, "error"); }else{ - taos_close(taos); + taos_close(taos); lua_pushnumber(L, 0); lua_setfield(L, table_index, "code"); lua_pushstring(L, "done."); diff --git a/tests/examples/lua/test.lua b/tests/examples/lua/test.lua index 4d5f9fe7d36f99f9a9452e8137bf298361a05685..9f9c6934aa46d52e1578700b067193351120dbab 100644 --- a/tests/examples/lua/test.lua +++ b/tests/examples/lua/test.lua @@ -1,93 +1,117 @@ local driver = require "luaconnector" -local host="127.0.0.1" -local user="root" -local password="taosdata" -local db =nil -local port=6030 -local conn +local config = { + host = "127.0.0.1", + port = 6030, + database = "", + user = "root", + password = "taosdata", + max_packet_size = 1024 * 1024 +} -local res = driver.connect(host,user,password,db,port) +local conn +local res = driver.connect(config) if res.code ~=0 then - print(res.error) + print("connect--- failed: "..res.error) return else conn = res.conn + print("connect--- pass.") end local res = driver.query(conn,"drop database if exists demo") res = driver.query(conn,"create database demo") if res.code ~=0 then - print(res.error) + print("create db--- failed: "..res.error) return +else + print("create db--- pass.") end res = driver.query(conn,"use demo") if res.code ~=0 then - print(res.error) + print("select db--- failed: "..res.error) return +else + print("select db--- pass.") end res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") if res.code ~=0 then - print(res.error) + print("create table---failed: "..res.error) return +else + print("create table--- pass.") end res = driver.query(conn,"insert into m1 values ('2019-09-01 00:00:00.001',0,'robotspace'), ('2019-09-01 00:00:00.002',1,'Hilink'),('2019-09-01 00:00:00.003',2,'Harmony')") if res.code ~=0 then - print(res.error) + print("insert records failed: "..res.error) return else - print("insert successfully, affected:"..res.affected) + if(res.affected == 3) then + print("insert records--- pass") + else + print("insert records---failed: expect 3 affected records, actually affected "..res.affected) + end end res = driver.query(conn,"select * from m1") if res.code ~=0 then - print("select error:"..res.error) + print("select failed: "..res.error) return else - print("in lua, result:") - for i = 1, #(res.item) do - print("timestamp:"..res.item[i].ts) - print("speed:"..res.item[i].speed) - print("owner:"..res.item[i].owner) - end + if (#(res.item) == 3) then + print("select--- pass") + else + print("select--- failed: expect 3 affected records, actually received "..#(res.item)) + end + end res = driver.query(conn,"CREATE TABLE thermometer (ts timestamp, degree double) TAGS(location binary(20), type int)") if res.code ~=0 then print(res.error) return +else + print("create super table--- pass") end res = driver.query(conn,"CREATE TABLE therm1 USING thermometer TAGS ('beijing', 1)") if res.code ~=0 then print(res.error) return +else + print("create table--- pass") end + res = driver.query(conn,"INSERT INTO therm1 VALUES ('2019-09-01 00:00:00.001', 20),('2019-09-01 00:00:00.002', 21)") if res.code ~=0 then print(res.error) return else - print("insert successfully, affected:"..res.affected) + if(res.affected == 2) then + print("insert records--- pass") + else + print("insert records---failed: expect 2 affected records, actually affected "..res.affected) + end end res = driver.query(conn,"SELECT COUNT(*) count, AVG(degree) AS av, MAX(degree), MIN(degree) FROM thermometer WHERE location='beijing' or location='tianjin' GROUP BY location, type") if res.code ~=0 then - print("select error:"..res.error) + print("select from super table--- failed:"..res.error) return else - print("in lua, result:") + print("select from super table--- pass") for i = 1, #(res.item) do print("res:"..res.item[i].count) end end function callback(t) + print("------------------------") print("continuous query result:") for key, value in pairs(t) do print("key:"..key..", value:"..value) @@ -97,25 +121,25 @@ end local stream res = driver.open_stream(conn,"SELECT COUNT(*) as count, AVG(degree) as avg, MAX(degree) as max, MIN(degree) as min FROM thermometer interval(2s) sliding(2s);)",0,callback) if res.code ~=0 then - print("open stream error:"..res.error) + print("open stream--- failed:"..res.error) return else - print("openstream ok") + print("open stream--- pass") stream = res.stream end ---From now on we begin continous query in an definite (infinite if you want) loop. +print("From now on we start continous insert in an definite (infinite if you want) loop.") local loop_index = 0 -while loop_index < 10 do +while loop_index < 30 do local t = os.time()*1000 local v = loop_index res = driver.query(conn,string.format("INSERT INTO therm1 VALUES (%d, %d)",t,v)) if res.code ~=0 then - print(res.error) + print("continous insertion--- failed:" .. res.error) return else - print("insert successfully, affected:"..res.affected) + --print("insert successfully, affected:"..res.affected) end os.execute("sleep " .. 1) loop_index = loop_index + 1 diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index b96daa54649b115961de97245c6cd109d82f0768..8498094181857095fa52123b364119801dc93d56 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -74,8 +74,14 @@ function runQueryPerfTest { CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` + AVGDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` + MAXDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'` + MINDELAY=`echo ${delay:0:${#delay}-2}` - python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND | tee -a $PERFORMANCE_TEST_REPORT + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT [ -f taosdemoperf.txt ] && rm taosdemoperf.txt } diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 17a08386928fcd3d632b39e12eeb9266f198fecd..7bdab8bc6717905e0d81309b0c948d2f3108b55c 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -17,6 +17,7 @@ import json import time import random import requests +import argparse from requests.auth import HTTPBasicAuth func_list=['avg','count','twa','sum','stddev','leastsquares','min', 'max','first','last','top','bottom','percentile','apercentile', @@ -32,19 +33,34 @@ condition_list=[ ] where_list = ['_c0>now-10d',' <50'," like \'%a%\'"] class ConcurrentInquiry: - def __init__(self,n_Therads=25,r_Therads=25): + # def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test', + # stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5, + # stableNum = 2,subtableNum = 1000,insertRows = 100): + def __init__(self,ts,host,user,password,dbname, + stb_prefix,subtb_prefix,n_Therads,r_Therads,probabilities,loop, + stableNum ,subtableNum ,insertRows ,mix_table): self.n_numOfTherads = n_Therads self.r_numOfTherads = r_Therads - self.ts=1500000001000 - self.dbname='test' + self.ts=ts + self.host = host + self.user = user + self.password = password + self.dbname=dbname + self.stb_prefix = stb_prefix + self.subtb_prefix = subtb_prefix self.stb_list=[] self.subtb_list=[] self.stb_stru_list=[] self.subtb_stru_list=[] self.stb_tag_list=[] self.subtb_tag_list=[] - self.probabilities = [0.95,0.05] + self.probabilities = [probabilities,1-probabilities] self.ifjoin = [0,1] + self.loop = loop + self.stableNum = stableNum + self.subtableNum = subtableNum + self.insertRows = insertRows + self.mix_table = mix_table def SetThreadsNum(self,num): self.numOfTherads=num @@ -88,9 +104,9 @@ class ConcurrentInquiry: self.subtb_tag_list.append(tag) def get_full(self): #获取所有的表、表结构 - host = "127.0.0.1" - user = "root" - password = "taosdata" + host = self.host + user = self.user + password = self.password conn = taos.connect( host, user, @@ -117,15 +133,15 @@ class ConcurrentInquiry: return 'where '+random.choice([' and ',' or ']).join(l) def con_interval(self,tlist,col_list,tag_list): - interval = 'interval' + str(random.randint(0,100)) + random.choice(['a','s','d','w','n','y']) + interval = 'interval(' + str(random.randint(0,20)) + random.choice(['a','s','d','w','n','y']) + ')' return interval def con_limit(self,tlist,col_list,tag_list): rand1 = str(random.randint(0,1000)) rand2 = str(random.randint(0,1000)) - return random.choice(['limit ' + rand1,'limit ' + rand1 + 'offset '+rand2, - 'slimit ' + rand1,'slimit ' + rand1 + 'offset ' + rand2,'limit '+rand1 + 'slimit '+ rand2, - 'limit '+ rand1 + 'offset' + rand2 + 'slimit '+ rand1 + 'soffset ' + rand2 ]) + return random.choice(['limit ' + rand1,'limit ' + rand1 + ' offset '+rand2, + ' slimit ' + rand1,' slimit ' + rand1 + ' offset ' + rand2,'limit '+rand1 + ' slimit '+ rand2, + 'limit '+ rand1 + ' offset' + rand2 + ' slimit '+ rand1 + ' soffset ' + rand2 ]) def con_fill(self,tlist,col_list,tag_list): return random.choice(['fill(null)','fill(prev)','fill(none)','fill(LINEAR)']) @@ -133,7 +149,7 @@ class ConcurrentInquiry: def con_group(self,tlist,col_list,tag_list): rand_tag = random.randint(0,5) rand_col = random.randint(0,1) - return 'group by '+','.join(random.sample(col_list,rand_col))+','.join(random.sample(tag_list,rand_tag)) + return 'group by '+','.join(random.sample(col_list,rand_col) + random.sample(tag_list,rand_tag)) def con_order(self,tlist,col_list,tag_list): return 'order by '+random.choice(tlist) @@ -165,8 +181,10 @@ class ConcurrentInquiry: random.shuffle(func_list) sel_col_list=[] col_rand=random.randint(0,len(col_list)) + loop = 0 for i,j in zip(col_list[0:col_rand],func_list): #决定每个被查询col的函数 - alias = 'as '+ str(i) + alias = ' as '+ 'taos%d ' % loop + loop += 1 pick_func = '' if j == 'leastsquares': pick_func=j+'('+i+',1,1)' @@ -178,14 +196,20 @@ class ConcurrentInquiry: pick_func+=alias sel_col_list.append(pick_func) - sql=sql+','.join(sel_col_list)+' from '+random.choice(self.stb_list+self.subtb_list)+' ' #select col & func + sql=sql+','.join(sel_col_list) #select col & func + if self.mix_table == 0: + sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' ' + elif self.mix_table == 1: + sql = sql + ' from '+random.choice(self.subtb_list)+' ' + else: + sql = sql + ' from '+random.choice(self.stb_list)+' ' con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill] sel_con=random.sample(con_func,random.randint(0,len(con_func))) sel_con_list=[] for i in sel_con: sel_con_list.append(i(tlist,col_list,tag_list)) #获取对应的条件函数 sql+=' '.join(sel_con_list) # condition - print(sql) + #print(sql) return sql def gen_query_join(self): #生成join查询语句 @@ -194,9 +218,25 @@ class ConcurrentInquiry: tag_list = [] col_intersection = [] tag_intersection = [] - - - if bool(random.getrandbits(1)): + subtable = None + if self.mix_table == 0: + if bool(random.getrandbits(1)): + subtable = True + tbname = random.sample(self.subtb_list,2) + for i in tbname: + col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) + tag_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) + col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + else: + tbname = random.sample(self.stb_list,2) + for i in tbname: + col_list.append(self.stb_stru_list[self.stb_list.index(i)]) + tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) + col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) + tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) + elif self.mix_table == 1: + subtable = True tbname = random.sample(self.subtb_list,2) for i in tbname: col_list.append(self.subtb_stru_list[self.subtb_list.index(i)]) @@ -210,12 +250,9 @@ class ConcurrentInquiry: tag_list.append(self.stb_stru_list[self.stb_list.index(i)]) col_intersection = list(set(col_list[0]).intersection(set(col_list[1]))) tag_intersection = list(set(tag_list[0]).intersection(set(tag_list[1]))) - - con_rand=random.randint(0,len(condition_list)) col_rand=random.randint(0,len(col_list)) tag_rand=random.randint(0,len(tag_list)) - sql='select ' #select sel_col_tag=[] @@ -227,15 +264,18 @@ class ConcurrentInquiry: sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1]))) sql += ','.join(sel_col_tag) - sql = sql + 'from '+ ','.join(tbname) + ' ' #select col & func - con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill] - sel_con=random.sample(con_func,random.randint(0,len(con_func))) - sel_con_list=[] - - # for i in sel_con: - # sel_con_list.append(i(tlist,col_list,tag_list)) #获取对应的条件函数 - sql+=' '.join(sel_con_list) # condition - print(sql) + sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func + join_section = None + temp = None + if subtable: + temp = random.choices(col_intersection) + join_section = temp.pop() + sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) + else: + temp = random.choices(col_intersection+tag_intersection) + join_section = temp.pop() + print(random.choices(col_intersection)) + sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) return sql def random_pick(self): @@ -246,16 +286,48 @@ class ConcurrentInquiry: if x < cumulative_probability:break return item - + def gen_data(self): + stableNum = self.stableNum + subtableNum = self.subtableNum + insertRows = self.insertRows + t0 = self.ts + host = self.host + user = self.user + password = self.password + conn = taos.connect( + host, + user, + password, + ) + cl = conn.cursor() + cl.execute("drop database if exists %s;" %self.dbname) + cl.execute("create database if not exists %s;" %self.dbname) + cl.execute("use %s" % self.dbname) + for k in range(stableNum): + sql="create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20)) \ + tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20))" % (self.stb_prefix+str(k)) + cl.execute(sql) + for j in range(subtableNum): + sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s')" % \ + (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j)) + print(sql) + cl.execute(sql) + for i in range(insertRows): + ret = cl.execute( + "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % + (self.subtb_prefix+str(k)+'_'+str(j),t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i),'涛思'+str(i))) + cl.close() + conn.close() + def rest_query(self,sql): #rest 接口 - host = "127.0.0.1" - user = "root" - password = "taosdata" + host = self.host + user = self.user + password = self.password port =6041 url = "http://{}:{}/rest/sql".format(host, port ) try: r = requests.post(url, - data = 'use test', + data = 'use %s' % self.dbname, auth = HTTPBasicAuth('root', 'taosdata')) r = requests.post(url, data = sql, @@ -285,20 +357,20 @@ class ConcurrentInquiry: def query_thread_n(self,threadID): #使用原生python接口查询 - host = "127.0.0.1" - user = "root" - password = "taosdata" + host = self.host + user = self.user + password = self.password conn = taos.connect( host, user, password, ) cl = conn.cursor() - cl.execute("use test;") + cl.execute("use %s;" % self.dbname) print("Thread %d: starting" % threadID) - - while True: + loop = self.loop + while loop: try: if self.random_pick(): @@ -312,33 +384,44 @@ class ConcurrentInquiry: end = time.time() print("time cost :",end-start) except Exception as e: + print('-'*40) print( - "Failure thread%d, sql: %s,exception: %s" % + "Failure thread%d, sql: %s \nexception: %s" % (threadID, str(sql),str(e))) - #exit(-1) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + loop -= 1 + if loop == 0: break - + cl.close() + conn.close() print("Thread %d: finishing" % threadID) def query_thread_r(self,threadID): #使用rest接口查询 print("Thread %d: starting" % threadID) - while True: - try: - if self.random_pick(): - sql=self.gen_query_sql() - else: - sql=self.gen_query_join() - print("sql is ",sql) - start = time.time() - self.rest_query(sql) - end = time.time() - print("time cost :",end-start) - except Exception as e: - print( - "Failure thread%d, sql: %s,exception: %s" % - (threadID, str(sql),str(e))) - #exit(-1) - + loop = self.loop + while loop: + try: + if self.random_pick(): + sql=self.gen_query_sql() + else: + sql=self.gen_query_join() + print("sql is ",sql) + start = time.time() + self.rest_query(sql) + end = time.time() + print("time cost :",end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql),str(e))) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + loop -= 1 + if loop == 0: break print("Thread %d: finishing" % threadID) @@ -353,10 +436,132 @@ class ConcurrentInquiry: thread = threading.Thread(target=self.query_thread_r, args=(i,)) threads.append(thread) thread.start() -if len(sys.argv)>1: - q = ConcurrentInquiry(n_Therads=sys.argv[1],r_Therads=sys.argv[2]) -else: - q = ConcurrentInquiry() + +parser = argparse.ArgumentParser() +parser.add_argument( + '-H', + '--host-name', + action='store', + default='127.0.0.1', + type=str, + help='host name to be connected (default: 127.0.0.1)') +parser.add_argument( + '-S', + '--ts', + action='store', + default=1500000000000, + type=int, + help='insert data from timestamp (default: 1500000000000)') +parser.add_argument( + '-d', + '--db-name', + action='store', + default='test', + type=str, + help='Database name to be created (default: test)') +parser.add_argument( + '-t', + '--number-of-native-threads', + action='store', + default=10, + type=int, + help='Number of native threads (default: 10)') +parser.add_argument( + '-T', + '--number-of-rest-threads', + action='store', + default=10, + type=int, + help='Number of rest threads (default: 10)') +parser.add_argument( + '-r', + '--number-of-records', + action='store', + default=100, + type=int, + help='Number of record to be created for each table (default: 100)') +parser.add_argument( + '-c', + '--create-table', + action='store', + default='0', + type=int, + help='whether gen data (default: 0)') +parser.add_argument( + '-p', + '--subtb-name-prefix', + action='store', + default='t', + type=str, + help='subtable-name-prefix (default: t)') +parser.add_argument( + '-P', + '--stb-name-prefix', + action='store', + default='st', + type=str, + help='stable-name-prefix (default: st)') +parser.add_argument( + '-b', + '--probabilities', + action='store', + default='0.05', + type=float, + help='probabilities of join (default: 0.05)') +parser.add_argument( + '-l', + '--loop-per-thread', + action='store', + default='100', + type=int, + help='loop per thread (default: 100)') +parser.add_argument( + '-u', + '--user', + action='store', + default='root', + type=str, + help='user name') +parser.add_argument( + '-w', + '--password', + action='store', + default='root', + type=str, + help='user name') +parser.add_argument( + '-n', + '--number-of-tables', + action='store', + default=1000, + type=int, + help='Number of subtales per stable (default: 1000)') +parser.add_argument( + '-N', + '--number-of-stables', + action='store', + default=2, + type=int, + help='Number of stables (default: 2)') +parser.add_argument( + '-m', + '--mix-stable-subtable', + action='store', + default=0, + type=int, + help='0:stable & substable ,1:subtable ,2:stable (default: 0)') + +args = parser.parse_args() +q = ConcurrentInquiry( + args.ts,args.host_name,args.user,args.password,args.db_name, + args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads, + args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records, + args.mix_stable_subtable ) + +if args.create_table: + q.gen_data() q.get_full() + #q.gen_query_sql() q.run() + diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index e2ce4b26fa2fa8cc9c601218c1bf19d838b57c46..c6b857b097671b8faf650f4659c2b2990ef67d97 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2224,22 +2224,25 @@ class ClientManager: if svcMgr: # gConfig.auto_start_service: svcMgr.stopTaosServices() svcMgr = None - # Print exec status, etc., AFTER showing messages from the server - self.conclude() - # print("TC failed (2) = {}".format(self.tc.isFailed())) - # Linux return code: ref https://shapeshed.com/unix-exit-codes/ - ret = 1 if self.tc.isFailed() else 0 - self.tc.cleanup() + # Release global variables gConfig = None gSvcMgr = None logger = None + + thPool = None + dbManager.cleanUp() # destructor wouldn't run in time + dbManager = None + # Print exec status, etc., AFTER showing messages from the server + self.conclude() + # print("TC failed (2) = {}".format(self.tc.isFailed())) + # Linux return code: ref https://shapeshed.com/unix-exit-codes/ + ret = 1 if self.tc.isFailed() else 0 + self.tc.cleanup() # Release variables here self.tc = None - thPool = None - dbManager = None gc.collect() # force garbage collection # h = hpy() diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 855e18be55fd5ae0d88865644a2ebfc836b7ccf3..dc072d7abce68debd69cb162a4d784fcc5b68c4e 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -394,6 +394,7 @@ class DbManager(): cType == 'native') else DbConn.createRest(dbTarget) try: self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected + Logging.debug("DbManager opened DB connection...") except taos.error.ProgrammingError as err: # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) if (err.msg == 'client disconnected'): # cannot open DB connection @@ -412,6 +413,10 @@ class DbManager(): # Moved to Database() # self._stateMachine = StateMechine(self._dbConn) + def __del__(self): + ''' Release the underlying DB connection upon deletion of DbManager ''' + self.cleanUp() + def getDbConn(self): return self._dbConn @@ -438,5 +443,8 @@ class DbManager(): return "table_{}".format(tblNum) def cleanUp(self): - self._dbConn.close() + if self._dbConn: + self._dbConn.close() + self._dbConn = None + Logging.debug("DbManager closed DB connection...") diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index a374ed943b41b52c2ccbee0825bca3080ed43ea9..6ea5691ce223eb1c14214d4b11c47cf85e29c795 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -2,6 +2,7 @@ import threading import random import logging import os +import sys import taos @@ -53,7 +54,7 @@ class Logging: # global misc.logger _logger = logging.getLogger('CrashGen') # real logger _logger.addFilter(LoggingFilter()) - ch = logging.StreamHandler() + ch = logging.StreamHandler(sys.stdout) # Ref: https://stackoverflow.com/questions/14058453/making-python-loggers-output-all-messages-to-stdout-in-addition-to-log-file _logger.addHandler(ch) # Logging adapter, to be used as a logger diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py index fd12284b9d7782ac7df89c37fcb653ca3bebe82b..de2d9b0780f3654424d8e077a4dda9ac5ac0ed50 100644 --- a/tests/pytest/crash_gen_bootstrap.py +++ b/tests/pytest/crash_gen_bootstrap.py @@ -19,5 +19,5 @@ if __name__ == "__main__": mExec.init() exitCode = mExec.run() - print("Exiting with code: {}".format(exitCode)) + print("\nCrash_Gen is now exiting with status code: {}".format(exitCode)) sys.exit(exitCode) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 664dbd6e5608ad53c4e19492bf73e2ca7505407c..cb233f85b8132910b316fd130983769e99452bf9 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -19,7 +19,7 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py -python3 ./test.py -f insert/before_1970.py +#python3 ./test.py -f insert/before_1970.py python3 bug2265.py #table @@ -34,7 +34,6 @@ python3 ./test.py -f table/alter_column.py python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py -python3 ./test.py -f table/queryWithTaosdKilled.py # tag @@ -167,18 +166,27 @@ python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug2218.py python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2118.py python3 ./test.py -f query/bug2143.py python3 ./test.py -f query/sliding.py python3 ./test.py -f query/unionAllTest.py python3 ./test.py -f query/bug2281.py python3 ./test.py -f query/bug2119.py +python3 ./test.py -f query/isNullTest.py +python3 ./test.py -f query/queryWithTaosdKilled.py +python3 ./test.py -f query/floatCompare.py + #stream python3 ./test.py -f stream/metric_1.py +python3 ./test.py -f stream/metric_n.py python3 ./test.py -f stream/new.py python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py #python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py +python3 ./test.py -f stream/sys.py +python3 ./test.py -f stream/table_1.py +python3 ./test.py -f stream/table_n.py #alter table python3 ./test.py -f alter/alter_table_crash.py @@ -211,13 +219,16 @@ python3 ./test.py -f functions/function_spread.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 -#python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py +python3 ./test.py -f functions/all_null_value.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py python3 client/twoClients.py -python3 test.py -f query/queryInterval.py -python3 test.py -f query/queryFillTest.py +python3 ./test.py -f query/queryInterval.py +python3 ./test.py -f query/queryFillTest.py +python3 ./test.py -f query/last_row_cache.py +python3 ./test.py -f query/last_cache.py # tools python3 test.py -f tools/taosdemoTest.py diff --git a/tests/pytest/functions/all_null_value.py b/tests/pytest/functions/all_null_value.py new file mode 100644 index 0000000000000000000000000000000000000000..5354b48f80dae25f7cd65fb00f90e4eeef2908cc --- /dev/null +++ b/tests/pytest/functions/all_null_value.py @@ -0,0 +1,90 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 int, c2 int)") + for i in range(self.rowNum): + tdSql.execute("insert into st values(%d, null, null)" % (self.ts + i)) + + tdSql.query("select avg(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select max(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select min(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select bottom(c1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select top(c1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select diff(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select first(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select last(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select last_row(c1) from st") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select count(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select leastsquares(c1, 1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select c1 + c2 from st") + tdSql.checkRows(10) + + tdSql.query("select spread(c1) from st") + tdSql.checkRows(0) + + # tdSql.query("select stddev(c1) from st") + # tdSql.checkRows(0) + + tdSql.query("select sum(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select twa(c1) from st") + tdSql.checkRows(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index 4074166f927acbbcbb5899a692ff87f3c187f92f..3cc389221845b38d92ca130e086d50fddc344d6d 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -85,6 +85,22 @@ class TDTestCase: tdSql.checkData(0, 1, 0.1) tdSql.checkData(1, 1, 1.1) + #TD-2457 bottom + interval + order by + tdSql.error('select top(col2,1) from test interval(1y) order by col2;') + + #TD-2563 top + super_table + interval + tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") + tdSql.execute("create table t1 using meters tags (1)") + sql = 'insert into t1 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i , i % 47) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t1 values ' + tdSql.execute(sql) + tdSql.query('select bottom(c,1) from meters interval(10a)') + tdSql.checkData(0,1,0) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py index 3bd5031276adfd76574a1a43e311a1d6ec81ab16..d1f8f75892ce2e61388c107eb7cad1ed00582d2f 100644 --- a/tests/pytest/functions/function_max.py +++ b/tests/pytest/functions/function_max.py @@ -69,6 +69,15 @@ class TDTestCase: tdSql.query("select max(col6) from test1") tdSql.checkData(0, 0, np.max(floatData)) + + # test case: https://jira.taosdata.com:18080/browse/TD-2583 + tdSql.execute("create database test days 2") + tdSql.execute("create table car(ts timestamp, speed int)") + tdSql.execute("insert into car values(now, -1)") + tdSql.execute("insert into car values(now-10d, null)") + + tdSql.query("select max(speed) from car") + tdSql.checkData(0, 0, -1) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py index bc180bc22414a4a3aa3d233438b10a04fd08bc2b..c779744ced63d83fd5b2116fb61dfa347ec4dd5a 100644 --- a/tests/pytest/functions/function_min.py +++ b/tests/pytest/functions/function_min.py @@ -69,6 +69,15 @@ class TDTestCase: tdSql.query("select min(col6) from test1") tdSql.checkData(0, 0, np.min(floatData)) + + # test case: https://jira.taosdata.com:18080/browse/TD-2583 + tdSql.execute("create database test days 2") + tdSql.execute("create table car(ts timestamp, speed int)") + tdSql.execute("insert into car values(now, 1)") + tdSql.execute("insert into car values(now-10d, null)") + + tdSql.query("select min(speed) from car") + tdSql.checkData(0, 0, 1) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py index e63d65f2e6a429015e2b4d7dcbe5e8c9884eea5e..e793008ce681a04111fdee29e91520fed944d05a 100644 --- a/tests/pytest/functions/function_percentile.py +++ b/tests/pytest/functions/function_percentile.py @@ -142,6 +142,14 @@ class TDTestCase: tdSql.error("select percentile(voltage, 20) from meters") tdSql.query("select apercentile(voltage, 20) from meters") print("apercentile result: %s" % tdSql.getData(0, 0)) + + # Test case for: https://jira.taosdata.com:18080/browse/TD-2609 + tdSql.execute("create table st(ts timestamp, k int)") + tdSql.execute("insert into st values(now, -100)") + tdSql.query("select apercentile(k, 20) from st") + tdSql.checkData(0, 0, -100.00) + + def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py index e24ff1cc5339c79e1a87653e9b528ec34f3c2ea9..a98f31eea0f48114be00d3ca93bb93a1622d8470 100644 --- a/tests/pytest/functions/function_top.py +++ b/tests/pytest/functions/function_top.py @@ -89,6 +89,20 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 8.1) tdSql.checkData(1, 1, 9.1) + + #TD-2563 top + super_table + interval + tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") + tdSql.execute("create table t1 using meters tags (1)") + sql = 'insert into t1 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i , i % 47) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t1 values ' + tdSql.execute(sql) + tdSql.query('select top(c,1) from meters interval(10a)') + tdSql.checkData(0,1,9) + def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_twa.py b/tests/pytest/functions/function_twa.py index 1ce4c99b6041e2cfd6bccdbc0e00e5fd8037f9d9..b9519a2abc4929ba2b459944a52d0ab34df57a6b 100644 --- a/tests/pytest/functions/function_twa.py +++ b/tests/pytest/functions/function_twa.py @@ -46,23 +46,17 @@ class TDTestCase: tdSql.error("select twa(ts) from test") tdSql.error("select twa(ts) from test1") - tdSql.error("select twa(col1) from test") - tdSql.error("select twa(col1) from test1") + tdSql.error("select twa(col1) from test") tdSql.error("select twa(col2) from test") - tdSql.error("select twa(col2) from test1") tdSql.error("select twa(col3) from test") - tdSql.error("select twa(col3) from test1") - tdSql.error("select twa(col4) from test") - tdSql.error("select twa(col4) from test1") + tdSql.error("select twa(col4) from test") tdSql.error("select twa(col5) from test") - tdSql.error("select twa(col5) from test1") - tdSql.error("select twa(col6) from test") - tdSql.error("select twa(col6) from test1") + tdSql.error("select twa(col6) from test") tdSql.error("select twa(col7) from test") tdSql.error("select twa(col7) from test1") @@ -72,59 +66,23 @@ class TDTestCase: tdSql.error("select twa(col9) from test") tdSql.error("select twa(col9) from test1") - - tdSql.error("select twa(col1) from test where ts > %d" % self.ts) - tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col2) from test where ts > %d" % self.ts) - tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col3) from test where ts > %d" % self.ts) - tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col4) from test where ts > %d" % self.ts) - tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col5) from test where ts > %d" % self.ts) - tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col6) from test where ts > %d" % self.ts) - tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) def stop(self): diff --git a/tests/pytest/functions/function_twa_restart.py b/tests/pytest/functions/function_twa_restart.py index 2025e3c9ddcdd41aa18db949e5db0d059090e469..56242c29535646f1baf12948ad06ebbe1a7f08ac 100644 --- a/tests/pytest/functions/function_twa_restart.py +++ b/tests/pytest/functions/function_twa_restart.py @@ -28,12 +28,17 @@ class TDTestCase: self.ts = 1537146000000 def run(self): - tdSql.execute("use db") + tdSql.prepare() intData = [] floatData = [] + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -41,23 +46,17 @@ class TDTestCase: tdSql.error("select twa(ts) from test") tdSql.error("select twa(ts) from test1") - tdSql.error("select twa(col1) from test") - tdSql.error("select twa(col1) from test1") + tdSql.error("select twa(col1) from test") tdSql.error("select twa(col2) from test") - tdSql.error("select twa(col2) from test1") tdSql.error("select twa(col3) from test") - tdSql.error("select twa(col3) from test1") - tdSql.error("select twa(col4) from test") - tdSql.error("select twa(col4) from test1") + tdSql.error("select twa(col4) from test") tdSql.error("select twa(col5) from test") - tdSql.error("select twa(col5) from test1") - tdSql.error("select twa(col6) from test") - tdSql.error("select twa(col6) from test1") + tdSql.error("select twa(col6) from test") tdSql.error("select twa(col7) from test") tdSql.error("select twa(col7) from test1") @@ -67,59 +66,23 @@ class TDTestCase: tdSql.error("select twa(col9) from test") tdSql.error("select twa(col9) from test1") - - tdSql.error("select twa(col1) from test where ts > %d" % self.ts) - tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col2) from test where ts > %d" % self.ts) - tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col3) from test where ts > %d" % self.ts) - tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col4) from test where ts > %d" % self.ts) - tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col5) from test where ts > %d" % self.ts) - tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col6) from test where ts > %d" % self.ts) - tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) - - tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) - tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) - - tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) - tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) def stop(self): @@ -127,4 +90,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/functions/function_twa_test2.py b/tests/pytest/functions/function_twa_test2.py index 744075c00967ee12f37f65cb3184cc7a1ad7b2ea..b20f14357e22d38162b5d69cc3ae643f700f8754 100644 --- a/tests/pytest/functions/function_twa_test2.py +++ b/tests/pytest/functions/function_twa_test2.py @@ -39,6 +39,10 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 5.5) + tdSql.query("select twa(c) from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 5.5) + tdSql.query("select twa(c) from t1 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(10s)") tdSql.checkRows(10) tdSql.checkData(0, 1, 1.49995) @@ -75,7 +79,75 @@ class TDTestCase: tdSql.checkData(18, 1, 9.75000) tdSql.checkData(19, 1, 10) + tdSql.execute("create table t2(ts timestamp, c int)") + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + 3000)) + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(2s) ") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 09:01:30.000' interval(2s) sliding(1s) ") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 1) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 09:00:04.000' and ts <= '2018-09-17 09:01:30.000' ") + tdSql.checkRows(0) + + tdSql.query("select twa(c) from t2 where ts >= '2018-09-17 08:00:00.000' and ts <= '2018-09-17 09:00:00.000' ") + tdSql.checkRows(0) + + tdSql.execute("create table t3(ts timestamp, c int)") + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts)) + tdSql.execute("insert into t3 values(%d, -2)" % (self.ts + 3000)) + + tdSql.query("select twa(c) from t3 where ts >= '2018-09-17 08:59:00.000' and ts <= '2018-09-17 09:01:30.000'") + tdSql.checkRows(1) + tdSql.checkData(0, 0, -0.5) + + tdSql.query("select twa(c) from t3 where ts >= '2018-09-17 08:59:00.000' and ts <= '2018-09-17 09:01:30.000' interval(1s)") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.5005) + tdSql.checkData(1, 1, -2) + + tdSql.query("select twa(c) from t3 where ts >= '2018-09-17 08:59:00.000' and ts <= '2018-09-17 09:01:30.000' interval(2s) sliding(1s)") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 0.5005) + tdSql.checkData(1, 1, 0.0005) + tdSql.checkData(2, 1, -1.5) + tdSql.checkData(3, 1, -2) + + #TD-2533 twa+interval with large records + tdSql.execute("create table t4(ts timestamp, c int)") + sql = 'insert into t4 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i * 500, i + 1) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t4 values ' + tdSql.execute(sql) + tdSql.query('select twa(c) from t4 interval(10s)') + tdSql.checkData(0,1,10.999) + + # Test case: https://jira.taosdata.com:18080/browse/TD-2624 + tdSql.execute("create database test keep 7300") + tdSql.execute("use test") + tdSql.execute("create table st(ts timestamp, k int)") + tdSql.execute("insert into st values('2011-01-02 18:42:45.326', -1)") + tdSql.execute("insert into st values('2020-07-30 17:44:06.283', 0)") + tdSql.execute("insert into st values('2020-07-30 17:44:19.578', 9999999)") + tdSql.execute("insert into st values('2020-07-30 17:46:06.417', NULL)") + tdSql.execute("insert into st values('2020-11-09 18:42:25.538', 0)") + tdSql.execute("insert into st values('2020-12-29 17:43:11.641', 0)") + tdSql.execute("insert into st values('2020-12-29 18:43:17.129', 0)") + tdSql.execute("insert into st values('2020-12-29 18:46:19.109', NULL)") + tdSql.execute("insert into st values('2021-01-03 18:40:40.065', 0)") + tdSql.query("select twa(k),first(ts) as taos1 from st where k <50 interval(17s)") + tdSql.checkRows(6) def stop(self): tdSql.close() diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index ce3d1c0c673e6e62fdd2e6ac83769eb3d7a8ca71..0f89ef2ec4b01f402a2d10bad829269c77b7c2ec 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -5,13 +5,24 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & -./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4 -pidof taosd|xargs kill +IN_TDINTERNAL="community" +TDIR=`pwd` +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../../.. +fi +TOP_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep -v community|head -n1` +nohup $TAOSD_DIR >/dev/null & +cd - +./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4 +pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` do +memError=(${memError//,/}) if [ -n "$memError" ]; then if [ "$memError" -gt 12 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ @@ -23,7 +34,7 @@ done grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'` do - +defiMemError=(${defiMemError//,/}) if [ -n "$defiMemError" ]; then if [ "$defiMemError" -gt 3 ]; then echo -e "${RED} ## Memory errors number valgrind reports \ @@ -31,4 +42,4 @@ if [ -n "$defiMemError" ]; then exit 8 fi fi -done \ No newline at end of file +done diff --git a/tests/pytest/import_merge/importCacheFileH.py b/tests/pytest/import_merge/importCacheFileH.py index 3398f7bdad8d7524e223f0a2847b579081686476..33724dfa6889903a34836844271f5843c7cb36e4 100644 --- a/tests/pytest/import_merge/importCacheFileH.py +++ b/tests/pytest/import_merge/importCacheFileH.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHO.py b/tests/pytest/import_merge/importCacheFileHO.py index 19520dc3d048a267c03850b608d47db20394c43d..9f552978f9bb5b0be72eb7cba7ebd073114e740b 100644 --- a/tests/pytest/import_merge/importCacheFileHO.py +++ b/tests/pytest/import_merge/importCacheFileHO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHPO.py b/tests/pytest/import_merge/importCacheFileHPO.py index 9e0a57fb462cdb6ae1dee3c8bfee4488c99a76a0..f2d0a3ddbfb4c0a2a111aab423e32c0ed4683e47 100644 --- a/tests/pytest/import_merge/importCacheFileHPO.py +++ b/tests/pytest/import_merge/importCacheFileHPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileS.py b/tests/pytest/import_merge/importCacheFileS.py index 2f0af569e564b64cbb88c048762e8d41dcf21cbd..250952b2377f9d164c16801b6ead5f695d4f7c43 100644 --- a/tests/pytest/import_merge/importCacheFileS.py +++ b/tests/pytest/import_merge/importCacheFileS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileSub.py b/tests/pytest/import_merge/importCacheFileSub.py index 300bb6e8d09f4484e22e64a0f2caf50cab7c8ab4..663bdcd6351c02a91da309502e0a7f5ec9878086 100644 --- a/tests/pytest/import_merge/importCacheFileSub.py +++ b/tests/pytest/import_merge/importCacheFileSub.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileT.py b/tests/pytest/import_merge/importCacheFileT.py index ab33cf6a93c4bbd7f27967f6a70b61fc62230f65..f2feeeacff11a4c8a2d65feaa08cd1b57264e0c9 100644 --- a/tests/pytest/import_merge/importCacheFileT.py +++ b/tests/pytest/import_merge/importCacheFileT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTO.py b/tests/pytest/import_merge/importCacheFileTO.py index 00e22da976618b4ee870a5702542f669693e3727..421ca6a755425690e7ec76d9a932ce11a65cb141 100644 --- a/tests/pytest/import_merge/importCacheFileTO.py +++ b/tests/pytest/import_merge/importCacheFileTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTPO.py b/tests/pytest/import_merge/importCacheFileTPO.py index c6089e1d687fea6ac900931a8171e45f4923f0cb..ca582d7d5aad8106d7c3d12439001fda84d5004c 100644 --- a/tests/pytest/import_merge/importCacheFileTPO.py +++ b/tests/pytest/import_merge/importCacheFileTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataH2.py b/tests/pytest/import_merge/importDataH2.py index a21f0c47be2eb11a57182d22b56d4447ff2d7ab6..4de567d943a1fe843ec16dd4589054ebcef76488 100644 --- a/tests/pytest/import_merge/importDataH2.py +++ b/tests/pytest/import_merge/importDataH2.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO.py b/tests/pytest/import_merge/importDataHO.py index fdcaedd83c99482914805cb65a067cbcbc78a128..f40de71d6d3967aac6d851deae2810aed63bc3c8 100644 --- a/tests/pytest/import_merge/importDataHO.py +++ b/tests/pytest/import_merge/importDataHO.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO2.py b/tests/pytest/import_merge/importDataHO2.py index b094701132677d6e72e834b16b7edc106682be14..518975e0b727db0d51bd6f5b68e690356988e2fe 100644 --- a/tests/pytest/import_merge/importDataHO2.py +++ b/tests/pytest/import_merge/importDataHO2.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHPO.py b/tests/pytest/import_merge/importDataHPO.py index 9d74c0c3522eba3687618132c27ce95f86f91b91..c7f0625d4bf09de49066c9ea19d4936bdd00e811 100644 --- a/tests/pytest/import_merge/importDataHPO.py +++ b/tests/pytest/import_merge/importDataHPO.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastH.py b/tests/pytest/import_merge/importDataLastH.py index c8e5f62423dc21254d215fac41926aeb9ce5ba40..f736cb925a405c8588d01faad0b08759127962b4 100644 --- a/tests/pytest/import_merge/importDataLastH.py +++ b/tests/pytest/import_merge/importDataLastH.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHO.py b/tests/pytest/import_merge/importDataLastHO.py index 33215997a4220b65228b630ebc350bf9dae15046..b1eebfdd47ed1fbb5185b7547927ea50a29bc64e 100644 --- a/tests/pytest/import_merge/importDataLastHO.py +++ b/tests/pytest/import_merge/importDataLastHO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHPO.py b/tests/pytest/import_merge/importDataLastHPO.py index fa8542f35b8e9639db20dc2e1886fca36140e825..9aa8127f50799039ceaa48dee957cdb903b890f0 100644 --- a/tests/pytest/import_merge/importDataLastHPO.py +++ b/tests/pytest/import_merge/importDataLastHPO.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastS.py b/tests/pytest/import_merge/importDataLastS.py index 2f595fef542955e616ef2e6ebcce7f23bce71d80..2c1559d2903a4fde104603bb2ad5945ea8eddd1e 100644 --- a/tests/pytest/import_merge/importDataLastS.py +++ b/tests/pytest/import_merge/importDataLastS.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastSub.py b/tests/pytest/import_merge/importDataLastSub.py index bfcad2d252cccb2404b3989c474310e0a19afe2e..415d1fcba27921267832323e6c3177678a215c9b 100644 --- a/tests/pytest/import_merge/importDataLastSub.py +++ b/tests/pytest/import_merge/importDataLastSub.py @@ -32,7 +32,7 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.deploy(1) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') @@ -60,9 +60,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 10 data totally repetitive") diff --git a/tests/pytest/import_merge/importDataLastT.py b/tests/pytest/import_merge/importDataLastT.py index 08e944eb6836a6f7f938639b40a6124f7bb64ec6..839320cc22352f1009d0d786aa58cf8827fc8ff6 100644 --- a/tests/pytest/import_merge/importDataLastT.py +++ b/tests/pytest/import_merge/importDataLastT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTO.py b/tests/pytest/import_merge/importDataLastTO.py index a82c0541414298accfdfb447c318ffafd88e5852..0c35519fcf76e7d2faab3a2cb2c4cc702fe93f0b 100644 --- a/tests/pytest/import_merge/importDataLastTO.py +++ b/tests/pytest/import_merge/importDataLastTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTPO.py b/tests/pytest/import_merge/importDataLastTPO.py index ff75a1b2ae87891853fa93f0f49fff339cf6e488..c7864ae14d68a3ef8860125920b3286921f65356 100644 --- a/tests/pytest/import_merge/importDataLastTPO.py +++ b/tests/pytest/import_merge/importDataLastTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataS.py b/tests/pytest/import_merge/importDataS.py index 37627e8d6bcc8319805275525ef7874132e0da4d..f0d56a5016fe5b059e1b51719b799d801219c2ca 100644 --- a/tests/pytest/import_merge/importDataS.py +++ b/tests/pytest/import_merge/importDataS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataSub.py b/tests/pytest/import_merge/importDataSub.py index 17e2b141b7014fd2ce4d31f8c335e122aca730ba..1643cc89155507be6c5fa2c072755a8439b21aab 100644 --- a/tests/pytest/import_merge/importDataSub.py +++ b/tests/pytest/import_merge/importDataSub.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataT.py b/tests/pytest/import_merge/importDataT.py index b0b7b82b790e81c475c714ba570a9a6f1801cc13..c23ad9a60533d94d34d11c70ea2743a1b8ab5c01 100644 --- a/tests/pytest/import_merge/importDataT.py +++ b/tests/pytest/import_merge/importDataT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTO.py b/tests/pytest/import_merge/importDataTO.py index c0b57136af7a51c1d184eb5789d95efc0148118c..b4c0d1a8adb6d4a3751d5a4a6547d0d03eab793b 100644 --- a/tests/pytest/import_merge/importDataTO.py +++ b/tests/pytest/import_merge/importDataTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTPO.py b/tests/pytest/import_merge/importDataTPO.py index 8a1c9264b433484e768cdfffbccdb0966e3d15cd..38e8cfe9096135f1f9301d89bc8c6269d045e78f 100644 --- a/tests/pytest/import_merge/importDataTPO.py +++ b/tests/pytest/import_merge/importDataTPO.py @@ -57,9 +57,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 20 data later with partly overlap") diff --git a/tests/pytest/import_merge/importHORestart.py b/tests/pytest/import_merge/importHORestart.py index f74c4c76d616753f8cadc87ba715e473eae0fb46..05b5b42afff14c695690b7651bd385ee78f3bafb 100644 --- a/tests/pytest/import_merge/importHORestart.py +++ b/tests/pytest/import_merge/importHORestart.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHPORestart.py b/tests/pytest/import_merge/importHPORestart.py index e5f79fbe6c2eb686d63f5b7de81651a98072a986..f9167709130d67b2909d3e9efd1acc482638836d 100644 --- a/tests/pytest/import_merge/importHPORestart.py +++ b/tests/pytest/import_merge/importHPORestart.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHRestart.py b/tests/pytest/import_merge/importHRestart.py index be67039789dd660ddb0be8d56753737010e47e79..c1d50378b20a11b4d562b408d7bec07364969d19 100644 --- a/tests/pytest/import_merge/importHRestart.py +++ b/tests/pytest/import_merge/importHRestart.py @@ -54,7 +54,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importInsertThenImport.py b/tests/pytest/import_merge/importInsertThenImport.py index 292fae8c47b398dcbacdb8b76592826bcc8b6019..1372177de36c1f67ab85ba4a63c26553ee4a7de4 100644 --- a/tests/pytest/import_merge/importInsertThenImport.py +++ b/tests/pytest/import_merge/importInsertThenImport.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.stop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdLog.info("import 100 sequential data again") diff --git a/tests/pytest/import_merge/importLastH.py b/tests/pytest/import_merge/importLastH.py index 17fa233e3723e1d9e6cc4ec97c8eac2716751f00..2887c71e040b7ea89e42da69e597bcb843a2df5c 100644 --- a/tests/pytest/import_merge/importLastH.py +++ b/tests/pytest/import_merge/importLastH.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHO.py b/tests/pytest/import_merge/importLastHO.py index adb44fc0eab9a4923b0896a1357b24603f1ff3fd..1681212b6b8c5d002e6f3d2b5853534382bb4d10 100644 --- a/tests/pytest/import_merge/importLastHO.py +++ b/tests/pytest/import_merge/importLastHO.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHPO.py b/tests/pytest/import_merge/importLastHPO.py index d8ed2d9ef10dbaa52e92ba50625be25c495091ea..389b3c11ed04b0359564a41c7b383e37e9bcabdd 100644 --- a/tests/pytest/import_merge/importLastHPO.py +++ b/tests/pytest/import_merge/importLastHPO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastS.py b/tests/pytest/import_merge/importLastS.py index bf222a0d5f3c27c5c7774b384799a6a0aa5e4114..e6393bbfae2f41efd1ff740f4e542715f23f8adf 100644 --- a/tests/pytest/import_merge/importLastS.py +++ b/tests/pytest/import_merge/importLastS.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastSub.py b/tests/pytest/import_merge/importLastSub.py index 5a6b9f41502970149257b1aef5c7a37824401768..4ff6cf27bdf582040360e40ce13d28df7dfc8949 100644 --- a/tests/pytest/import_merge/importLastSub.py +++ b/tests/pytest/import_merge/importLastSub.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastT.py b/tests/pytest/import_merge/importLastT.py index 2b1be1fe2b37aa5e4d59bb6ecef7d0bb2180cf9d..145b1bd69097804728020ea1b3b8a0ccf7e86d27 100644 --- a/tests/pytest/import_merge/importLastT.py +++ b/tests/pytest/import_merge/importLastT.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTO.py b/tests/pytest/import_merge/importLastTO.py index ce189f63718956991982bd33296eb71bd5a82d45..a159dac4acea9c1c278025e157ff3fe09487e858 100644 --- a/tests/pytest/import_merge/importLastTO.py +++ b/tests/pytest/import_merge/importLastTO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTPO.py b/tests/pytest/import_merge/importLastTPO.py index 627d090855ac320f56a90fb647101e8ccc315acd..453b922a68bc7a051aa60d51943ffb759444ee5b 100644 --- a/tests/pytest/import_merge/importLastTPO.py +++ b/tests/pytest/import_merge/importLastTPO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importSRestart.py b/tests/pytest/import_merge/importSRestart.py index 29f5a19b54fb0cbf1eb81a66c66813f3fcdd9982..58f0f60e0af0660e4550966c2f0b1741aefbc33c 100644 --- a/tests/pytest/import_merge/importSRestart.py +++ b/tests/pytest/import_merge/importSRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importSubRestart.py b/tests/pytest/import_merge/importSubRestart.py index b1a6f30c436c5aa52d5ee4affa03d18c5475ced7..85594e1772409f7c24a5d2ce8aa25b6ba7567a5d 100644 --- a/tests/pytest/import_merge/importSubRestart.py +++ b/tests/pytest/import_merge/importSubRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTORestart.py b/tests/pytest/import_merge/importTORestart.py index 07eb6c28cb79c16504f2acab036a3b66c3bccabf..10ac77d759192de329ab0818b79610259d05b7d7 100644 --- a/tests/pytest/import_merge/importTORestart.py +++ b/tests/pytest/import_merge/importTORestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTPORestart.py b/tests/pytest/import_merge/importTPORestart.py index 10bbf3efced90b6c255326c0e8b8e0c106a3a038..ab86a0247f179bd6dab397a069ca9b4aaf4da61f 100644 --- a/tests/pytest/import_merge/importTPORestart.py +++ b/tests/pytest/import_merge/importTPORestart.py @@ -68,7 +68,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTRestart.py b/tests/pytest/import_merge/importTRestart.py index 63a9368eca2b6a3377b32dc6a3948cf5a0ba1186..6106bdd75310369d53ce8148f937655301b64c12 100644 --- a/tests/pytest/import_merge/importTRestart.py +++ b/tests/pytest/import_merge/importTRestart.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index da797f788fcc3cc1ae06843b70da9aaf7230e97c..6489fd31ffd6f595d15d996b92f5c63fc7e2b8a9 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -38,7 +38,7 @@ class RestfulInsert: for i in range(loop): tableID = threadID * tablesPerThread if tableID + i >= self.numOfTables : break - name = 'beijing' if tableID % 2 == 0 else 'shanghai' + name = 'beijing' if (tableID + i) % 2 == 0 else 'shanghai' data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) response = requests.post(self.url, data, headers = self.header) if response.status_code != 200: diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index ebb24d76bf4680d0a5efbaabf844a4077ad28d57..ad26c480048e04e8a0955f908c0396d63726513e 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -19,6 +19,8 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py +#python3 ./test.py -f insert/before_1970.py +python3 bug2265.py #table python3 ./test.py -f table/alter_wal0.py @@ -32,7 +34,7 @@ python3 ./test.py -f table/alter_column.py python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py -python3 ./test.py -f table/queryWithTaosdKilled.py + # tag python3 ./test.py -f tag_lite/filter.py @@ -162,17 +164,29 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py -python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2118.py +python3 ./test.py -f query/bug2143.py +python3 ./test.py -f query/sliding.py +python3 ./test.py -f query/unionAllTest.py python3 ./test.py -f query/bug2281.py python3 ./test.py -f query/bug2119.py -python3 bug2265.py +python3 ./test.py -f query/isNullTest.py +python3 ./test.py -f query/queryWithTaosdKilled.py +python3 ./test.py -f query/floatCompare.py + #stream python3 ./test.py -f stream/metric_1.py +python3 ./test.py -f stream/metric_n.py python3 ./test.py -f stream/new.py python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py #python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py +python3 ./test.py -f stream/sys.py +python3 ./test.py -f stream/table_1.py +python3 ./test.py -f stream/table_n.py #alter table python3 ./test.py -f alter/alter_table_crash.py @@ -181,6 +195,7 @@ python3 ./test.py -f alter/alter_table_crash.py python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py +python3 ./test.py -f client/noConnectionErrorTest.py # Misc python3 testCompress.py @@ -204,7 +219,7 @@ python3 ./test.py -f functions/function_spread.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 -#python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py @@ -216,10 +231,10 @@ python3 test.py -f query/queryFillTest.py python3 test.py -f tools/taosdemoTest.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/lowaTest.py +python3 test.py -f tools/taosdemoTest2.py # subscribe python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py +#python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py - diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh index fededea3bb34481aa34c37d4a802ba183494e491..4ec517a0bf1c5eff8ad670cf28ab63d5ce818460 100755 --- a/tests/pytest/pytest_2.sh +++ b/tests/pytest/pytest_2.sh @@ -12,6 +12,10 @@ python3 ./test.py -f update/merge_commit_data2.py python3 ./test.py -f update/merge_commit_data2_update0.py python3 ./test.py -f update/merge_commit_last-0.py python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py # wal -python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file +python3 ./test.py -f wal/addOldWalTest.py + +# function +python3 ./test.py -f functions/all_null_value.py \ No newline at end of file diff --git a/tests/pytest/query/bug2118.py b/tests/pytest/query/bug2118.py new file mode 100644 index 0000000000000000000000000000000000000000..803f442a1b201b8fcd508e39f5f02b0ab95812aa --- /dev/null +++ b/tests/pytest/query/bug2118.py @@ -0,0 +1,52 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdLog.info("==========step1") + tdLog.info("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20))") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % + (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41))) + tdLog.info("==========step2") + tdLog.info("test percentile with group by normal_col ") + tdSql.query('select percentile(c1,1),percentile(c2,1),percentile(c6,1) from mt0 group by c3 limit 3 offset 2') + + tdSql.checkData(0,0,2.48) + tdSql.checkData(0,0,2.48) + tdSql.checkData(0,2,11.84) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/floatCompare.py b/tests/pytest/query/floatCompare.py new file mode 100644 index 0000000000000000000000000000000000000000..cdbddb893e8c0af19fc4aee1596172ade759583f --- /dev/null +++ b/tests/pytest/query/floatCompare.py @@ -0,0 +1,172 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table st(ts timestamp, c1 float, c2 double) tags(t1 float, t2 double)") + tdSql.execute( + 'CREATE TABLE if not exists t1 using st tags(1.023, 3.00412)') + tdSql.execute( + 'CREATE TABLE if not exists t2 using st tags(2.005, 10.129001)') + + print("==============step2") + for i in range(10): + tdSql.execute("insert into t1 values(%d, %s, %s) t2 values(%d, %s, %s)" % + (self.ts + i, 1.0001 + i, 2.0001301 + i, self.ts + i, 1.0001 * i, 2.0001301 * i)) + + tdSql.query("select count(*) from st") + tdSql.checkData(0, 0, 20) + + tdSql.query("select count(*) from st where t1 > 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 < 1.023") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t1 >= 1.023") + tdSql.checkData(0, 0, 20) + + tdSql.query("select count(*) from st where t1 <= 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 = 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 <> 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 != 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 >= 1.023 and t1 <= 1.023") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t1 > 1.023 and t1 <= 1.023") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t1 >= 1.023 and t1 < 1.023") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t2 > 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 < 3.00412") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t2 >= 3.00412") + tdSql.checkData(0, 0, 20) + + tdSql.query("select count(*) from st where t2 <= 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 = 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 <> 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 != 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 >= 3.00412 and t2 <= 3.00412") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(*) from st where t2 > 3.00412 and t2 <= 3.00412") + tdSql.checkRows(0) + + tdSql.query("select count(*) from st where t2 >= 3.00412 and t2 < 3.00412") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t1 where c1 < 2.0001") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t1 where c1 <= 2.0001") + tdSql.checkData(0, 0, 2) + + tdSql.query("select count(*) from t1 where c1 = 2.0001") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t1 where c1 > 2.0001") + tdSql.checkData(0, 0, 8) + + tdSql.query("select count(*) from t1 where c1 >= 2.0001") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t1 where c1 <> 2.0001") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t1 where c1 != 2.0001") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t1 where c1 >= 2.0001 and c1 <= 2.0001") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t1 where c1 > 2.0001 and c1 <= 2.0001") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t1 where c1 >= 2.0001 and c1 < 2.0001") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t2 where c2 < 2.0001301") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t2 where c2 <= 2.0001301") + tdSql.checkData(0, 0, 2) + + tdSql.query("select count(*) from t2 where c2 = 2.0001301") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t2 where c2 > 2.0001301") + tdSql.checkData(0, 0, 8) + + tdSql.query("select count(*) from t2 where c2 >= 2.0001301") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t2 where c2 <> 2.0001301") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t2 where c2 != 2.0001301") + tdSql.checkData(0, 0, 9) + + tdSql.query("select count(*) from t2 where c2 >= 2.0001301 and c2 <= 2.0001301") + tdSql.checkData(0, 0, 1) + + tdSql.query("select count(*) from t2 where c2 > 2.0001301 and c2 <= 2.0001301") + tdSql.checkRows(0) + + tdSql.query("select count(*) from t2 where c2 >= 2.0001301 and c2 < 2.0001301") + tdSql.checkRows(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/isNullTest.py b/tests/pytest/query/isNullTest.py new file mode 100644 index 0000000000000000000000000000000000000000..7b79679c7d9d9ac4629a69b32acb1a11b61a83c1 --- /dev/null +++ b/tests/pytest/query/isNullTest.py @@ -0,0 +1,128 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table st(ts timestamp, c1 int, c2 binary(20), c3 nchar(20)) tags(t1 int, t2 binary(20), t3 nchar(20))") + tdSql.execute("create table t1 using st tags(1, 'binary1', 'nchar1')") + tdSql.execute("insert into t2(ts, c2) using st(t2) tags('') values(%d, '')" % (self.ts + 10)) + tdSql.execute("insert into t3(ts, c2) using st(t3) tags('') values(%d, '')" % (self.ts + 10)) + + for i in range(10): + tdSql.execute("insert into t1 values(%d, %d, 'binary%d', 'nchar%d')" % (self.ts + i, i, i, i)) + tdSql.execute("insert into t2 values(%d, %d, 'binary%d', 'nchar%d')" % (self.ts + i, i, i, i)) + tdSql.execute("insert into t3 values(%d, %d, 'binary%d', 'nchar%d')" % (self.ts + i, i, i, i)) + + tdSql.execute("insert into t1(ts, c2) values(%d, '')" % (self.ts + 10)) + tdSql.execute("insert into t1(ts, c3) values(%d, '')" % (self.ts + 11)) + tdSql.execute("insert into t2(ts, c3) values(%d, '')" % (self.ts + 11)) + tdSql.execute("insert into t3(ts, c3) values(%d, '')" % (self.ts + 11)) + + tdSql.query("select count(*) from st") + tdSql.checkData(0, 0, 36) + + tdSql.query("select count(*) from st where t1 is null") + tdSql.checkData(0, 0, 24) + + tdSql.query("select count(*) from st where t1 is not null") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t2 is null") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t2 is not null") + tdSql.checkData(0, 0, 24) + + tdSql.error("select count(*) from st where t2 <> null") + tdSql.error("select count(*) from st where t2 = null") + + tdSql.query("select count(*) from st where t2 = '' ") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t2 <> '' ") + tdSql.checkData(0, 0, 24) + + tdSql.query("select count(*) from st where t3 is null") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t3 is not null") + tdSql.checkData(0, 0, 24) + + tdSql.error("select count(*) from st where t3 <> null") + tdSql.error("select count(*) from st where t3 = null") + + tdSql.query("select count(*) from st where t3 = '' ") + tdSql.checkData(0, 0, 12) + + tdSql.query("select count(*) from st where t3 <> '' ") + tdSql.checkData(0, 0, 24) + + tdSql.query("select count(*) from st where c1 is not null") + tdSql.checkData(0, 0, 30) + + tdSql.query("select count(*) from st where c1 is null") + tdSql.checkData(0, 0, 6) + + tdSql.query("select count(*) from st where c2 is not null") + tdSql.checkData(0, 0, 33) + + tdSql.query("select count(*) from st where c2 is null") + tdSql.checkData(0, 0, 3) + + tdSql.error("select count(*) from st where c2 <> null") + tdSql.error("select count(*) from st where c2 = null") + + tdSql.query("select count(*) from st where c2 = '' ") + tdSql.checkData(0, 0, 3) + + tdSql.query("select count(*) from st where c2 <> '' ") + tdSql.checkData(0, 0, 30) + + tdSql.query("select count(*) from st where c3 is not null") + tdSql.checkData(0, 0, 33) + + tdSql.query("select count(*) from st where c3 is null") + tdSql.checkData(0, 0, 3) + + tdSql.error("select count(*) from st where c3 <> null") + tdSql.error("select count(*) from st where c3 = null") + + tdSql.query("select count(*) from st where c3 = '' ") + tdSql.checkData(0, 0, 3) + + tdSql.query("select count(*) from st where c3 <> '' ") + tdSql.checkData(0, 0, 30) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/last_cache.py b/tests/pytest/query/last_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..c31d9821e2695dd7883674519325c24f0e473bf2 --- /dev/null +++ b/tests/pytest/query/last_cache.py @@ -0,0 +1,133 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.tables = 10 + self.rows = 20 + self.perfix = 't' + self.ts = 1601481600000 + + def insertData(self): + print("==============step1") + tdSql.execute("create table st (ts timestamp, c1 int) tags(t1 int)") + + for i in range(self.tables): + tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) + for j in range(self.rows): + tc = self.ts + j * 60000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, i, tc, j)) + + def executeQueries(self): + print("==============step2") + tdSql.query("select last(c1) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) from %s%d where ts <= %d" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) as b from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) from %s%d interval(1m)" % (self.perfix, 1)) + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from %s%d interval(1d)" % (self.perfix, 1)) + tdSql.checkData(0, 1, 19) + + tdSql.query("select last(c1) from %s%d where ts <= %d interval(1m)" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkRows(5) + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from st") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) as c from st where t1 <= 5") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where t1 <= 5 and ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) from st interval(1m)") + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from st interval(1d)") + tdSql.checkData(0, 1, 19) + + tdSql.query("select last(c1) from st group by t1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where ts <= %d interval(1m) group by t1" % (self.ts + 4 * 60000)) + tdSql.checkData(1, 1, 1) + tdSql.checkRows(50) + + def run(self): + tdSql.prepare() + + # last_cache_0.sim + tdSql.execute("create database test1 cachelast 0") + tdSql.execute("use test1") + self.insertData() + self.executeQueries() + + tdSql.execute("alter database test1 cachelast 1") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + tdSql.execute("alter database test1 cachelast 0") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + # last_cache_1.sim + tdSql.execute("create database test2 cachelast 1") + tdSql.execute("use test2") + self.insertData() + self.executeQueries() + + tdSql.execute("alter database test2 cachelast 0") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + tdSql.execute("alter database test2 cachelast 1") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..d9e09dae7acdcd8c7401f1370da9f544c850872d --- /dev/null +++ b/tests/pytest/query/last_row_cache.py @@ -0,0 +1,186 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.tables = 10 + self.rows = 20 + self.perfix = 't' + self.ts = 1601481600000 + + def insertData(self): + print("==============step1") + tdSql.execute("create table st (ts timestamp, c1 int) tags(t1 int)") + + for i in range(self.tables): + tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) + for j in range(self.rows): + tc = self.ts + j * 60000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, i, tc, j)) + + def executeQueries(self): + print("==============step2") + tdSql.query("select last_row(c1) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) from %s%d where ts <= %d" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as b from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) from st") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) as c from st where ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as c from st where t1 < 5") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) as c from st where t1 <= 5 and ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as c from st group by t1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 19) + + tc = self.ts + 1 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + + tc = self.ts + 3 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + tc = self.ts + 5 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + + tc = self.ts + 7 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + def insertData2(self): + tc = self.ts + 1 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + + tc = self.ts + 3 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + tc = self.ts + 5 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + + tc = self.ts + 7 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + def executeQueries2(self): + # For stable + tc = self.ts + 6 * 3600000 + tdSql.query("select last_row(c1) from st where ts < %d " % tc) + tdSql.checkData(0, 0, -1) + + tc = self.ts + 8 * 3600000 + tdSql.query("select last_row(*) from st where ts < %d " % tc) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(*) from st") + tdSql.checkData(0, 1, None) + + tc = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts < %d " % tc) + tdSql.checkData(0, 1, None) + + tc1 = self.ts + 1 * 3600000 + tc2 = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts > %d and ts <= %d" % (tc1, tc2)) + tdSql.checkData(0, 1, None) + + # For table + tc = self.ts + 6 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, -1) + + tc = self.ts + 8 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(*) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 1, None) + + tc = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, None) + + tc1 = self.ts + 1 * 3600000 + tc2 = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts > %d and ts <= %d" % (tc1, tc2)) + tdSql.checkData(0, 1, None) + + def run(self): + tdSql.prepare() + + print("============== last_row_cache_0.sim") + tdSql.execute("create database test1 cachelast 0") + tdSql.execute("use test1") + self.insertData() + self.executeQueries() + self.insertData2() + self.executeQueries2() + + print("============== alter last cache") + tdSql.execute("alter database test1 cachelast 1") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test1 cachelast 0") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + print("============== last_row_cache_1.sim") + tdSql.execute("create database test2 cachelast 1") + tdSql.execute("use test2") + self.insertData() + self.executeQueries() + self.insertData2() + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 0") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 1") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index da0ef96d0efde77b716ded06cc7efaa96ea3bb7b..87635f86f39fb317907f689307eeefa9f3aef124 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -16,7 +16,7 @@ import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql - +from util.dnodes import tdDnodes class TDTestCase: def init(self, conn, logSql): @@ -44,6 +44,25 @@ class TDTestCase: tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'") tdSql.checkRows(1) + ## test case for https://jira.taosdata.com:18080/browse/TD-2488 + tdSql.execute("create table m1(ts timestamp, k int) tags(a int)") + tdSql.execute("create table t1 using m1 tags(1)") + tdSql.execute("create table t2 using m1 tags(2)") + tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)") + tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)") + tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)") + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryError.py b/tests/pytest/query/queryError.py index f1fd9c0dec480d8d4e0b49635fd23b275eeeaa24..539ce5141fae9cfba7ba63b569179c0170ef428d 100644 --- a/tests/pytest/query/queryError.py +++ b/tests/pytest/query/queryError.py @@ -56,6 +56,15 @@ class TDTestCase: # query .. order by non-time field tdSql.error("select * from st order by name") + # TD-2133 + tdSql.error("select diff(tagtype),bottom(tagtype,1) from dev_001") + + # TD-2190 + tdSql.error("select min(tagtype),max(tagtype) from dev_002 interval(1n) fill(prev)") + + # TD-2208 + tdSql.error("select diff(tagtype),top(tagtype,1) from dev_001") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 871c076c0813f0ccd340d33bff992401e522df4a..9cc468b34e6133af393d9d0d800e64d14e85c430 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -24,7 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.ts = 1593548685000 + self.ts = 1593548685000 def run(self): tdSql.prepare() @@ -84,6 +84,22 @@ class TDTestCase: tdDnodes.start(1) tdSql.query("select last(*) from t interval(1s)") tdSql.checkRows(10000) + + # test case for https://jira.taosdata.com:18080/browse/TD-2601 + newTs = 1601481600000 + + tdSql.execute("create database test2") + tdSql.execute("use test2") + tdSql.execute("create table t (ts timestamp, voltage int)") + for i in range(100): + tdSql.execute("insert into t values(%d, %d)" % (newTs + i * 10000000, i)) + + tdSql.query("select sum(voltage) from t where ts >='2020-10-01 00:00:00' and ts <='2020-12-01 00:00:00' interval(1n) fill(NULL)") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 4950) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, None) + def stop(self): diff --git a/tests/pytest/query/queryNullValueTest.py b/tests/pytest/query/queryNullValueTest.py index bc0b11827e18b0b436edac1cee2f04bafb05fab3..9920543b3a8d78363197a632eabe47ac6f82fa2e 100644 --- a/tests/pytest/query/queryNullValueTest.py +++ b/tests/pytest/query/queryNullValueTest.py @@ -50,7 +50,7 @@ class TDTestCase: tdSql.execute("insert into t0 values (%d, NULL)" % (self.ts)) tdDnodes.stop(1) - tdLog.sleep(10) + # tdLog.sleep(10) tdDnodes.start(1) tdSql.execute("use db") tdSql.query("select * from t0") @@ -62,7 +62,7 @@ class TDTestCase: tdSql.execute("create table t1 (ts timestamp, col %s)" % self.types[i]) tdSql.execute("insert into t1 values (%d, NULL)" % (self.ts)) tdDnodes.stop(1) - tdLog.sleep(10) + # tdLog.sleep(10) tdDnodes.start(1) tdSql.execute("use db") diff --git a/tests/pytest/query/queryWithTaosdKilled.py b/tests/pytest/query/queryWithTaosdKilled.py new file mode 100644 index 0000000000000000000000000000000000000000..28f9b87636987559669952a5fa88c25963fa9388 --- /dev/null +++ b/tests/pytest/query/queryWithTaosdKilled.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + self.conn = conn + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def createOldDir(self): + path = tdDnodes.dnodes[1].getDnodeRootDir(1) + print(path) + tdLog.info("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) + os.system("sudo mkdir -p %s/data/vnode/vnode2/wal/old" % path) + + def run(self): + # os.system("rm -rf %s/ " % tdDnodes.getDnodesRootDir()) + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, speed int)") + tdSql.execute("insert into st values(now, 1)") + tdSql.query("select count(*) from st") + tdSql.checkRows(1) + + + self.createOldDir() + tdLog.sleep(10) + + print("force kill taosd") + os.system("sudo kill -9 $(pgrep -x taosd)") + os.system("") + tdDnodes.start(1) + + tdSql.init(self.conn.cursor()) + tdSql.execute("use db") + tdSql.query("select count(*) from st") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/metric_n.py b/tests/pytest/stream/metric_n.py new file mode 100644 index 0000000000000000000000000000000000000000..d223fe81fc79835047bac8ca2341cdbeac2e6617 --- /dev/null +++ b/tests/pytest/stream/metric_n.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tbNum = 10 + rowNum = 20 + totalNum = tbNum * rowNum + + tdSql.prepare() + + tdLog.info("===== preparing data =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step 1 =====") + tdSql.query("select count(*), count(tbcol), count(tbcol2) from stb interval(1d)") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 2 =====") + tdSql.execute("create table strm_c3 as select count(*), count(tbcol), count(tbcol2) from stb interval(1d)") + + tdLog.info("===== step 3 =====") + tdSql.execute("create table strm_c32 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + + tdLog.info("===== step 4 =====") + tdSql.query("select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 5 =====") + tdSql.execute("create table strm_c31 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + + tdLog.info("===== step 6 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.execute("create table strm_avg as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb interval(1d)") + + tdLog.info("===== step 7 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from stb where ts < now + 4m interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, totalNum) + + tdLog.info("===== step 8 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from stb where ts < now + 4m interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, totalNum) + + tdLog.info("===== step 9 =====") + tdSql.waitedQuery("select * from strm_c3", 1, 120) + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 10 =====") + tdSql.waitedQuery("select * from strm_c31", 1, 30) + for i in range(1, 10): + tdSql.checkData(0, i, totalNum) + + tdLog.info("===== step 11 =====") + tdSql.waitedQuery("select * from strm_avg", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/stream2.py b/tests/pytest/stream/stream2.py index 44882f59721c4967ebb3fea12e356bb7e95d71ae..d71742048a45774fbe6909093840f265b829eb4c 100644 --- a/tests/pytest/stream/stream2.py +++ b/tests/pytest/stream/stream2.py @@ -87,6 +87,10 @@ class TDTestCase: tdSql.checkData(0, 3, rowNum) except Exception as e: tdLog.info(repr(e)) + + tdSql.query("show streams") + tdSql.checkRows(1) + tdSql.checkData(0, 2, 's0') tdLog.info("===== step8 =====") tdSql.query( @@ -142,6 +146,12 @@ class TDTestCase: except Exception as e: tdLog.info(repr(e)) + tdSql.query("show streams") + tdSql.checkRows(2) + tdSql.checkData(0, 2, 's1') + tdSql.checkData(1, 2, 's0') + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/stream/sys.py b/tests/pytest/stream/sys.py new file mode 100644 index 0000000000000000000000000000000000000000..a73e7043e8c65b2eb9c78fbcb99d4e546ddf9ae4 --- /dev/null +++ b/tests/pytest/stream/sys.py @@ -0,0 +1,62 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# migrated from 'stream_on_sys.sim' +# -*- coding: utf-8 -*- +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + updatecfgDict = {'monitor': 1} + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def run(self): + time.sleep(5) + tdSql.execute("use log") + + tdSql.execute("create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s)") + tdSql.execute("create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s)") + tdSql.execute("create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s)") + tdSql.execute("create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s)") + tdSql.execute("create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s)") + tdSql.execute("create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s)") + + sqls = [ + "select * from cpustrm", + "select * from memstrm", + "select * from diskstrm", + "select * from bandstrm", + "select * from reqstrm", + "select * from iostrm", + ] + for sql in sqls: + (rows, _) = tdSql.waitedQuery(sql, 1, 120) + if rows < 1: + tdLog.exit("failed: sql:%s, expect at least one row" % sql) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/stream/table_1.py b/tests/pytest/stream/table_1.py new file mode 100644 index 0000000000000000000000000000000000000000..a9fd57393112177fe0e290cadeaabd97d669daca --- /dev/null +++ b/tests/pytest/stream/table_1.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def createFuncStream(self, expr, suffix, value): + tbname = "strm_" + suffix + tdLog.info("create stream table %s" % tbname) + tdSql.query("select %s from tb1 interval(1d)" % expr) + tdSql.checkData(0, 1, value) + tdSql.execute("create table %s as select %s from tb1 interval(1d)" % (tbname, expr)) + + def checkStreamData(self, suffix, value): + sql = "select * from strm_" + suffix + tdSql.waitedQuery(sql, 1, 120) + tdSql.checkData(0, 1, value) + + def run(self): + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + self.createFuncStream("count(*)", "c1", rowNum) + self.createFuncStream("count(tbcol)", "c2", rowNum) + self.createFuncStream("count(tbcol2)", "c3", rowNum) + self.createFuncStream("avg(tbcol)", "av", 9.5) + self.createFuncStream("sum(tbcol)", "su", 190) + self.createFuncStream("min(tbcol)", "mi", 0) + self.createFuncStream("max(tbcol)", "ma", 19) + self.createFuncStream("first(tbcol)", "fi", 0) + self.createFuncStream("last(tbcol)", "la", 19) + self.createFuncStream("stddev(tbcol)", "st", 5.766281297335398) + self.createFuncStream("percentile(tbcol, 1)", "pe", 0.19) + self.createFuncStream("count(tbcol)", "as", rowNum) + + self.checkStreamData("c1", rowNum) + self.checkStreamData("c2", rowNum) + self.checkStreamData("c3", rowNum) + self.checkStreamData("av", 9.5) + self.checkStreamData("su", 190) + self.checkStreamData("mi", 0) + self.checkStreamData("ma", 19) + self.checkStreamData("fi", 0) + self.checkStreamData("la", 19) + self.checkStreamData("st", 5.766281297335398) + self.checkStreamData("pe", 0.19) + self.checkStreamData("as", rowNum) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/table_n.py b/tests/pytest/stream/table_n.py new file mode 100644 index 0000000000000000000000000000000000000000..371af769778bce1eb1e6cf1bac89333006c582a8 --- /dev/null +++ b/tests/pytest/stream/table_n.py @@ -0,0 +1,143 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== preparing data =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step 1 =====") + tdSql.query("select count(*), count(tbcol), count(tbcol2) from tb1 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 2 =====") + tdSql.execute("create table strm_c3 as select count(*), count(tbcol), count(tbcol2) from tb1 interval(1d)") + + tdLog.info("===== step 3 =====") + tdSql.execute("create table strm_c32 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + + tdLog.info("===== step 4 =====") + tdSql.query("select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 5 =====") + tdSql.execute("create table strm_c31 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + + tdLog.info("===== step 6 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from tb1 interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.execute("create table strm_avg as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from tb1 interval(1d)") + + tdLog.info("===== step 7 =====") + tdSql.query("select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from tb1 interval(1d)") + tdSql.checkData(0, 1, 5.766281297335398) + tdSql.checkData(0, 3, 0.19) + tdSql.execute("create table strm_ot as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from tb1 interval(1d)") + + tdLog.info("===== step 8 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, 5.766281297335398) + tdSql.checkData(0, 8, 0.19) + tdSql.checkData(0, 9, rowNum) + tdSql.execute("create table strm_to as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 interval(1d)") + + tdLog.info("===== step 9 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 where ts < now + 4m interval(1d)") + tdSql.checkData(0, 9, rowNum) + tdSql.execute("create table strm_wh as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 where ts < now + 4m interval(1d)") + + tdLog.info("===== step 10 =====") + tdSql.waitedQuery("select * from strm_c3", 1, 120) + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 11 =====") + tdSql.waitedQuery("select * from strm_c31", 1, 30) + for i in range(1, 10): + tdSql.checkData(0, i, rowNum) + + tdLog.info("===== step 12 =====") + tdSql.waitedQuery("select * from strm_avg", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + + tdLog.info("===== step 13 =====") + tdSql.waitedQuery("select * from strm_ot", 1, 20) + tdSql.checkData(0, 1, 5.766281297335398) + tdSql.checkData(0, 3, 0.19) + + tdLog.info("===== step 14 =====") + tdSql.waitedQuery("select * from strm_to", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, 5.766281297335398) + tdSql.checkData(0, 8, 0.19) + tdSql.checkData(0, 9, rowNum) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/alter_wal0.py b/tests/pytest/table/alter_wal0.py index 15ad69998f450b8e385cbf58052d246d9de27380..807dd316f2f5b154817217c9cf28821242bc9176 100644 --- a/tests/pytest/table/alter_wal0.py +++ b/tests/pytest/table/alter_wal0.py @@ -43,7 +43,8 @@ class TDTestCase: print("==============step2") tdDnodes.stopAll() - filename = '/var/lib/taos/mnode/wal/wal0' + path = tdDnodes.getDnodesRootDir() + filename = path + '/dnode1/data/mnode/wal/wal0' with open(filename, 'rb') as f1: temp = f1.read() @@ -57,12 +58,12 @@ class TDTestCase: print("==============step3") tdSql.execute("use demo;") tdSql.query('show tables;') - tdSql.checkRows(10) + tdSql.checkRows(9) for i in range(11,21): tdSql.execute("CREATE table if not exists test{num} using meters tags({num});".format(num=i)) tdSql.query('show tables;') - tdSql.checkRows(20) + tdSql.checkRows(19) print("==============check table numbers and create 10 tables") diff --git a/tests/pytest/table/create.py b/tests/pytest/table/create.py index 8fedd4e92046f58a94db66a1e696edfe5d253982..a0991d674a23065cbec5ae3b69d6bdf37ee363ac 100644 --- a/tests/pytest/table/create.py +++ b/tests/pytest/table/create.py @@ -39,6 +39,23 @@ class TDTestCase: except Exception as e: tdLog.exit(e) + # case for defect: https://jira.taosdata.com:18080/browse/TD-2560 + tdSql.execute("create table db.tb02 using st tags(2)") + tdSql.execute("create table db.tb03 using st tags(3)") + tdSql.execute("create table db.tb04 using st tags(4)") + + tdSql.query("show tables like 'tb%' ") + tdSql.checkRows(4) + + tdSql.query("show tables like 'tb0%' ") + tdSql.checkRows(3) + + tdSql.execute("create table db.st0 (ts timestamp, i int) tags(j int)") + tdSql.execute("create table db.st1 (ts timestamp, i int, c2 int) tags(j int, loc nchar(20))") + + tdSql.query("show stables like 'st%' ") + tdSql.checkRows(3) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index a2c35444b55fea9df55e46b4c81a9150b642b452..6be86fe3fdc4f0cd781d7749b9011a72ad687571 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -111,13 +111,25 @@ if __name__ == "__main__": tdLog.info('stop All dnodes') sys.exit(0) - + tdDnodes.init(deployPath) tdDnodes.setTestCluster(testCluster) tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() - tdDnodes.deploy(1) + is_test_framework = 0 + key_word = 'tdCases.addLinux' + if key_word in open(fileName).read(): + is_test_framework = 1 + if is_test_framework: + moduleName = fileName.replace(".py", "").replace("/", ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + tdDnodes.deploy(1,{}) tdDnodes.start(1) if masterIp == "": diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 6b6296e61a7965facd3a414ad635a4c64af43cc4..28f451b6a08a43508352c75b0c498251c9afca54 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -22,12 +22,15 @@ import argparse import os.path class taosdemoPerformace: - def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond): + def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay): self.commitID = commitID self.dbName = dbName self.createTableTime = createTableTime self.insertRecordsTime = insertRecordsTime - self.recordsPerSecond = recordsPerSecond + self.recordsPerSecond = recordsPerSecond + self.avgDelay = avgDelay + self.maxDelay = maxDelay + self.minDelay = minDelay self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -43,12 +46,15 @@ class taosdemoPerformace: cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("use %s" % self.dbName) - cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50))") + cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") print("==================== taosdemo performance ====================") print("create tables time: %f" % self.createTableTime) print("insert records time: %f" % self.insertRecordsTime) print("records per second: %f" % self.recordsPerSecond) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s')" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID)) + print("avg delay: %f" % self.avgDelay) + print("max delay: %f" % self.maxDelay) + print("min delay: %f" % self.minDelay) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) cursor.execute("drop database if exists taosdemo_insert_test") cursor.close() @@ -86,8 +92,28 @@ if __name__ == '__main__': action='store', type=float, help='records per request') + parser.add_argument( + '-avg', + '---avg-delay', + action='store', + type=float, + help='avg delay') + parser.add_argument( + '-max', + '---max-delay', + action='store', + type=float, + help='max delay') + parser.add_argument( + '-min', + '---min-delay', + action='store', + type=float, + help='min delay') + args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second) + perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, + args.avg_delay, args.max_delay, args.min_delay) perftest.createTablesAndStoreData() \ No newline at end of file diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index 3169b748e0843c720beb54eced28a14f1ca747a6..867ee696a261936e0118250995da4da5dca7ffb5 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -38,38 +38,62 @@ class TDTestCase: insertRows = 200 t0 = 1604298064000 + sql='insert into db.t1 values ' + temp='' tdLog.info("insert %d rows" % (insertRows)) for i in range(0, insertRows): - ret = tdSql.execute( - 'insert into t1 values (%d , 1)' % - (t0+i)) + # ret = tdSql.execute( + # 'insert into t1 values (%d , 1)' % + # (t0+i)) + temp += '(%d,1)' %(t0+i) + if i % 100 == 0 or i == (insertRows - 1 ): + print(sql+temp) + ret = tdSql.execute( + sql+temp + ) + temp = '' print("==========step2") print("restart to commit ") tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows) + for k in range(0,100): tdLog.info("insert %d rows" % (insertRows)) + temp='' for i in range (0,insertRows): - ret = tdSql.execute( - 'insert into db.t1 values(%d,1)' % - (t0+k*200+i) - ) + temp += '(%d,1)' %(t0+k*200+i) + if i % 100 == 0 or i == (insertRows - 1 ): + print(sql+temp) + ret = tdSql.execute( + sql+temp + ) + temp = '' + tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows+200*k) - print("==========step2") + print("==========step3") print("insert into another table ") s = 'use db' tdSql.execute(s) ret = tdSql.execute('create table t2 (ts timestamp, a int)') insertRows = 20000 + sql = 'insert into t2 values ' + temp = '' for i in range(0, insertRows): - ret = tdSql.execute( - 'insert into t2 values (%d, 1)' % - (t0+i)) + # ret = tdSql.execute( + # 'insert into t2 values (%d, 1)' % + # (t0+i)) + temp += '(%d,1)' %(t0+i) + if i % 500 == 0 or i == (insertRows - 1 ): + print(sql+temp) + ret = tdSql.execute( + sql+temp + ) + temp = '' tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t2") diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 757399b4a262dff7b11619791d3c82686fb293e8..83cb4e8d9994b1729ee13dda6829de300d9e8d7a 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -108,6 +108,36 @@ class TDDnode: self.deployed = 0 self.testCluster = False self.valgrind = 0 + self.cfgDict = { + "numOfLogLines":"100000000", + "mnodeEqualVnodeNum":"0", + "walLevel":"2", + "fsync":"1000", + "statusInterval":"1", + "numOfMnodes":"3", + "numOfThreadsPerCore":"2.0", + "monitor":"0", + "maxVnodeConnections":"30000", + "maxMgmtConnections":"30000", + "maxMeterConnections":"30000", + "maxShellConns":"30000", + "locale":"en_US.UTF-8", + "charset":"UTF-8", + "asyncLog":"0", + "anyIp":"0", + "tsEnableTelemetryReporting":"0", + "dDebugFlag":"135", + "mDebugFlag":"135", + "sdbDebugFlag":"135", + "rpcDebugFlag":"135", + "tmrDebugFlag":"131", + "cDebugFlag":"135", + "httpDebugFlag":"135", + "monitorDebugFlag":"135", + "udebugFlag":"135", + "jnidebugFlag":"135", + "qdebugFlag":"135" + } def init(self, path): self.path = path @@ -131,7 +161,10 @@ class TDDnode: return totalSize - def deploy(self): + def addExtraCfg(self, option, value): + self.cfgDict.update({option: value}) + + def deploy(self, *updatecfgDict): self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) @@ -175,36 +208,17 @@ class TDDnode: self.cfg("publicIp", "192.168.0.%d" % (self.index)) self.cfg("internalIp", "192.168.0.%d" % (self.index)) self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("tsEnableTelemetryReporting", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") + + self.cfg("dataDir",self.dataDir) + self.cfg("logDir",self.logDir) + print(updatecfgDict) + if updatecfgDict[0] and updatecfgDict[0][0]: + print(updatecfgDict[0][0]) + for key,value in updatecfgDict[0][0].items(): + self.addExtraCfg(key,value) + for key, value in self.cfgDict.items(): + self.cfg(key, value) + self.deployed = 1 tdLog.debug( "dnode:%d is deployed and configured by %s" % @@ -255,9 +269,33 @@ class TDDnode: tdLog.exit(cmd) self.running = 1 tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + if self.valgrind == 0: + time.sleep(0.1) + key = 'from offline to online' + bkey = bytes(key,encoding="utf8") + logFile = self.logDir + "/taosdlog.0" + i = 0 + while not os.path.exists(logFile): + sleep(0.1) + i += 1 + if i>50: + break + popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + pid = popen.pid + print('Popen.pid:' + str(pid)) + while True: + line = popen.stdout.readline().strip() + if bkey in line: + print(line) + popen.kill() + break + tdLog.debug("the dnode:%d has been started." % (self.index)) + else: + tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) + time.sleep(5) - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) + + # time.sleep(5) def startWithoutSleep(self): buildPath = self.getBuildPath() @@ -437,7 +475,7 @@ class TDDnodes: def setValgrind(self, value): self.valgrind = value - def deploy(self, index): + def deploy(self, index, *updatecfgDict): self.sim.setTestCluster(self.testCluster) if (self.simDeployed == False): @@ -447,7 +485,7 @@ class TDDnodes: self.check(index) self.dnodes[index - 1].setTestCluster(self.testCluster) self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() + self.dnodes[index - 1].deploy(updatecfgDict) def cfg(self, index, option, value): self.check(index) diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 4dd3957f390237c6dddc6b1f66b14ee177b6dbcf..cde51ebdbfc90f50bbe8d7941fd4f697d8b59ee5 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -134,8 +134,6 @@ run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim #unsupport run general/parser/slimit_alter_tags.sim -#unsupport run general/parser/stream_on_sys.sim -#unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim run general/stable/metrics.sim @@ -213,9 +211,6 @@ run general/vector/table_time.sim run general/stream/restart_stream.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/general/db/alter_tables_d2.sim b/tests/script/general/db/alter_tables_d2.sim index 9ef39fb55664ad0f984a3a2b4971daa69feec074..cd3121057b2c8d02755e89c40e11ca7b9d52b4ce 100644 --- a/tests/script/general/db/alter_tables_d2.sim +++ b/tests/script/general/db/alter_tables_d2.sim @@ -11,11 +11,26 @@ system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start - -sleep 3000 sql connect sql create dnode $hostname2 -sleep 1000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ============================ step1 @@ -80,10 +95,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -110,7 +138,6 @@ if $rows != 40 then return -1 endi - sql insert into db.t100 values(now, 1) sql insert into db.t101 values(now, 1) sql insert into db.t102 values(now, 1) @@ -144,10 +171,24 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -198,7 +239,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -268,10 +309,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -321,7 +375,7 @@ if $rows != 80 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -351,13 +405,26 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache -sleep 1000 +sleep 100 sql show db.tables if $rows != 80 then @@ -401,10 +468,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step1xx: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1xx +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1xx +endi sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) @@ -454,7 +534,7 @@ if $rows != 100 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then diff --git a/tests/script/general/db/alter_tables_v1.sim b/tests/script/general/db/alter_tables_v1.sim index dde5eb6d9ecfbce4467a54abf883cc6b9ffbb696..20c4c7336312b7eb701fe04c6b4158f466e43146 100644 --- a/tests/script/general/db/alter_tables_v1.sim +++ b/tests/script/general/db/alter_tables_v1.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -51,9 +49,22 @@ print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step2 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t10 using db.st tags(0) sql create table db.t11 using db.st tags(1) @@ -91,9 +102,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 30 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t20 using db.st tags(0) sql create table db.t21 using db.st tags(1) @@ -124,7 +148,7 @@ sql insert into db.t29 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t0 if $rows != 1 then @@ -148,9 +172,22 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step7 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step7 +endi sql reset query cache sleep 1000 @@ -190,9 +227,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 40 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t30 using db.st tags(0) sql create table db.t31 using db.st tags(1) @@ -249,12 +299,24 @@ if $rows != 40 then return -1 endi - print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -294,9 +356,22 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 50 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi sql create table db.t40 using db.st tags(0) sql create table db.t41 using db.st tags(1) diff --git a/tests/script/general/db/alter_tables_v4.sim b/tests/script/general/db/alter_tables_v4.sim index 7c9262874d8f07cef92df2585fe60261ea763080..b57b2c03208274949a901cf72c2d580133ca1ecd 100644 --- a/tests/script/general/db/alter_tables_v4.sim +++ b/tests/script/general/db/alter_tables_v4.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -70,9 +68,22 @@ endi print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -131,9 +142,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -184,7 +208,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -250,9 +274,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -331,9 +368,22 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -378,9 +428,24 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi + + sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) sql create table db.t402 using db.st tags(2) diff --git a/tests/script/general/db/delete_reusevnode.sim b/tests/script/general/db/delete_reusevnode.sim index 5781c384fe9bfd9ac804a3cd85a3e88a1e4bc278..4364a760a6b61be89f00f5a22d19c9ecf9015cb2 100644 --- a/tests/script/general/db/delete_reusevnode.sim +++ b/tests/script/general/db/delete_reusevnode.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======== step1 @@ -38,7 +37,6 @@ if $rows != 0 then endi system sh/stop_dnodes.sh -sleep 3000 system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes diff --git a/tests/script/general/db/delete_writing1.sim b/tests/script/general/db/delete_writing1.sim index 93d41b2fa37190e86133c47b087bc0a6a2cb71f1..8b369b4e3d0fc918aad19d2295583a0e2c1fda59 100644 --- a/tests/script/general/db/delete_writing1.sim +++ b/tests/script/general/db/delete_writing1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database db diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 94ecb59f75304d99f48ebdb644be432370f86f2a..7e12d30ac91eb49025fbebed05336e15cc6900ac 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -81,7 +81,7 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"data":[],"rows":0}@ then return -1 endi diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index 9851a4e7fcfea026271729cbe7436c47e184df40..405a8053120cd341773859bf14221ffb4c13dd7a 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -848,10 +848,7 @@ if $rows != 12 then return -1 endi -print =====================>td-1442 -sql_error select count(*) from m_fl_tb0 interval(1s) fill(prev); - -print =====================> aggregation + arithmetic + fill +print =====================> aggregation + arithmetic + fill, need to add cases TODO #sql select avg(cpu_taosd) - first(cpu_taosd) from dn1 where ts<'2020-11-13 11:00:00' and ts>'2020-11-13 10:50:00' interval(10s) fill(value, 99) #sql select count(*), first(k), avg(k), avg(k)-first(k) from tm0 where ts>'2020-1-1 1:1:1' and ts<'2020-1-1 1:02:59' interval(10s) fill(value, 99); #sql select count(*), first(k), avg(k), avg(k)-first(k) from tm0 where ts>'2020-1-1 1:1:1' and ts<'2020-1-1 1:02:59' interval(10s) fill(NULL); @@ -1044,6 +1041,17 @@ if $data12 != 1 then return -1 endi +print =====================>td-1442, td-2190 , no time range for fill option +sql_error select count(*) from m_fl_tb0 interval(1s) fill(prev); + +sql_error select min(c3) from m_fl_mt0 interval(10a) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10s) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10m) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10h) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10d) fill(value, 20) +sql_error select min(c3) from m_fl_mt0 interval(10w) fill(value, 20) +sql_error select max(c3) from m_fl_mt0 interval(1n) fill(prev) +sql_error select min(c3) from m_fl_mt0 interval(1y) fill(value, 20) print =============== clear #sql drop database $db diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index 773f92afcfd1fb126f137befd9de1107b050356f..df9a4598e0ba5438e45d299d467d86b2989eb825 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -106,7 +106,7 @@ while $x < 5000 endw system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 1000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 3505ad1a28139a85e62193eddeb050dfde0bbc0d..78b63c2baf460907a2d88836c52f157a8753c89c 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -244,3 +244,120 @@ if $data00 != -2.000000000 then print expect -2.000000000, actual: $data00 return -1 endi + +sql create table tm1 (ts timestamp, k int); +sql insert into tm1 values('2020-10-30 18:11:56.680', -1000); +sql insert into tm1 values('2020-11-19 18:11:45.773', NULL); +sql insert into tm1 values('2020-12-09 18:11:17.098', NULL); +sql insert into tm1 values('2020-12-20 18:11:49.412', 1); +sql insert into tm1 values('2020-12-23 18:11:50.412', 2); +sql insert into tm1 values('2020-12-28 18:11:52.412', 3); + +print =====================> td-2610 +sql select twa(k)from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-9 18:11:17.098' +if $rows != 0 then + return -1 +endi + +print =====================> td-2609 +sql select apercentile(k, 50) from tm1 where ts>='2020-10-30 18:11:56.680' and ts<='2020-12-09 18:11:17.098' +if $rows != 1 then + return -1 +endi + +if $data00 != -1000.000000000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 1000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect +sleep 500 + +sql use m_func_db0 + +print =====================> td-2583 +sql select min(k) from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-20 18:11:49.412' +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + print expect 1, actual: $data00 + return -1 +endi + +print =====================> td-2601 +sql select count(*) from tm1 where ts<='2020-6-1 00:00:00' and ts>='2020-1-1 00:00:00' interval(1n) fill(NULL) +if $rows != 0 then + return -1 +endi + +print =====================> td-2615 +sql select last(ts) from tm1 interval(17a) limit 776 offset 3 +if $rows != 3 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) limit 1000 offset 4 +if $rows != 2 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) order by ts desc limit 1000 offset 0 +if $rows != 6 then + return -1 +endi + +print ==================> td-2624 +sql create table tm2(ts timestamp, k int, b binary(12)); +sql insert into tm2 values('2011-01-02 18:42:45.326', -1,'abc'); +sql insert into tm2 values('2020-07-30 17:44:06.283', 0, null); +sql insert into tm2 values('2020-07-30 17:44:19.578', 9999999, null); +sql insert into tm2 values('2020-07-30 17:46:06.417', NULL, null); +sql insert into tm2 values('2020-11-09 18:42:25.538', 0, null); +sql insert into tm2 values('2020-12-29 17:43:11.641', 0, null); +sql insert into tm2 values('2020-12-29 18:43:17.129', 0, null); +sql insert into tm2 values('2020-12-29 18:46:19.109', NULL, null); +sql insert into tm2 values('2021-01-03 18:40:40.065', 0, null); + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s); +if $rows != 6 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data02 != @11-01-02 18:42:45.326@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi + +if $data21 != 0.000000000 then + return -1 +endi + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s) order by ts desc; +if $rows != 6 then + return -1 +endi + +sql select twa(k),first(ts),count(k),first(k) from tm2 interval(17s) limit 20 offset 0; +if $rows != 9 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi diff --git a/tests/script/general/parser/mixed_blocks.sim b/tests/script/general/parser/mixed_blocks.sim index d3558560df95d02256e0bc6b4a04ae9fae6e5890..946dbe8835dfcade3916754ebea12dd0d0c3a59d 100644 --- a/tests/script/general/parser/mixed_blocks.sim +++ b/tests/script/general/parser/mixed_blocks.sim @@ -144,4 +144,20 @@ if $data03 != 319 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print ===================> TD-2488 +sql create table m1(ts timestamp, k int) tags(a int); +sql create table t1 using m1 tags(1); +sql create table t2 using m1 tags(2); +sql insert into t1 values('2020-1-1 1:1:1', 1); +sql insert into t1 values('2020-1-1 1:10:1', 2); +sql insert into t2 values('2020-1-1 1:5:1', 99); + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql select ts from m1 where ts='2020-1-1 1:5:1' +if $rows != 1 then + return -1 +endi \ No newline at end of file diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim index bc22bd6da9194d9798f4744dea84d999beaf6a43..fc2ce1612313dad8636c13398695d87d0ddec0fc 100644 --- a/tests/script/general/parser/projection_limit_offset.sim +++ b/tests/script/general/parser/projection_limit_offset.sim @@ -317,7 +317,6 @@ sql_error select count(*) from group_tb0 where ts in ('2016-1-1 12:12:12'); sql_error select count(*) from group_tb0 where ts < '12:12:12'; #===============================sql for twa========================================== -sql_error select twa(c1) from group_tb0; sql_error select twa(c1) from group_stb0; sql_error select twa(c2) from group_stb0 where tsnow-1h group by t1; sql_error select twa(c2) from group_stb0 where tsnow-1h group by tbname,t1; @@ -393,7 +392,7 @@ endi #error sql sql_error select * from 1; -sql_error select 1; +#sql_error select 1; // equals to select server_status(); sql_error select k+k; sql_error select k+1; sql_error select abc(); diff --git a/tests/script/general/parser/repeatStream.sim b/tests/script/general/parser/repeatStream.sim deleted file mode 100644 index 616679e78bf8095e175ef91abf96c918626d6acb..0000000000000000000000000000000000000000 --- a/tests/script/general/parser/repeatStream.sim +++ /dev/null @@ -1,9 +0,0 @@ -$i = 1 -$repeats = 5 -while $i <= $repeats - print ====== repeat: $i - run general/parser/stream.sim - $i = $i + 1 -endw - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim deleted file mode 100644 index 1c8eb82c79a537ddbb322bdb164541263d92373c..0000000000000000000000000000000000000000 --- a/tests/script/general/parser/stream_on_sys.sim +++ /dev/null @@ -1,62 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 500 -sql connect -print ======================== stream_on_sys.sim - -$db = log -$tb = tb -$mt = mt -$strm = strm -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -$i = 0 - -sql use $db - -sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) -sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) -sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) -sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) -sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) -sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) -sleep 120000 -sql select * from cpustrm -if $rows <= 0 then - return -1 -endi - -sql select * from memstrm -if $rows <= 0 then - return -1 -endi - -sql select * from diskstrm -if $rows <= 0 then - return -1 -endi - -sql select * from bandstrm -if $rows <= 0 then - return -1 -endi - -sql select * from reqstrm -if $rows <= 0 then - return -1 -endi - -sql select * from iostrm -if $rows <= 0 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/tags_filter.sim b/tests/script/general/parser/tags_filter.sim index c3d0fdfc61d2d9ec8aa19100511577652076d91d..e05776ff11a67fb0b577219046044fdc8e825dc8 100644 --- a/tests/script/general/parser/tags_filter.sim +++ b/tests/script/general/parser/tags_filter.sim @@ -149,4 +149,57 @@ if $rows != 2 then return -1 endi +print ==================>td-2424 +sql create table t1(ts timestamp, k float) +sql insert into t1 values(now, 8.001) +sql select * from t1 where k=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k<8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k<=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k>8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k>=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k<>8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k>=8.001 and k<=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k>=8.0009999 and k<=8.001 +if $rows != 1 then + return -1 +endi + +sql select * from t1 where k>8.001 and k<=8.001 +if $rows != 0 then + return -1 +endi + +sql select * from t1 where k>=8.001 and k<8.001 +if $rows != 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index cea6d98679945b1d0e8d4dc82f3dd389b3d04a30..a2cf305fae890bf6685962d1c9817bf2a17c54c6 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -105,10 +105,3 @@ sleep 500 run general/parser/sliding.sim sleep 500 run general/parser/function.sim - -#sleep 500 -#run general/parser/repeatStream.sim -#sleep 500 -#run general/parser/stream_on_sys.sim -#sleep 500 -#run general/parser/stream.sim \ No newline at end of file diff --git a/tests/script/general/stream/metrics_n.sim b/tests/script/general/stream/metrics_n.sim deleted file mode 100644 index 7fc08064b29cc6f1681116be16ca460da0e41580..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/metrics_n.sim +++ /dev/null @@ -1,262 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = mn_db -$tbPrefix = mn_tb -$mtPrefix = mn_mt -$stPrefix = mn_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c3 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) -print select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) ===> $data00 $data01 $data02, $data03 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) - -print =============== step3 count32 -#total 32 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -# total 32 count in stream -$st = $stPrefix . c32 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -print =============== step4 count31 - -#total 31 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) -print ===> $data00 $data01 $data02, $data03 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -# total 31 count in stream will crash, 32 will error -$st = $stPrefix . c31 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -print =============== step5 avg ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . avg -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) - -print =============== step6 others -sql select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $mt interval(1d) -x step6 - return -1 -step6: - -$st = $stPrefix . ot -sql create table $st as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $mt interval(1d) -x step61 - return -1 -step61: - -print =============== step7 where -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 200 then - return -1 -endi - -$st = $stPrefix . wh -#sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) - -print =============== step8 as -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 200 then - return -1 -endi - -$st = $stPrefix . as -#sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7, avg(tbcol) as a8, sum(tbcol) as a9, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7 from $mt where ts < now + 4m interval(1d) - -print =============== step9 -print sleep 120 seconds -sleep 120000 - -print =============== step10 -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -$st = $stPrefix . c31 -sql select * from $st -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi -if $data04 != 200 then - return -1 -endi -if $data05 != 200 then - return -1 -endi -if $data06 != 200 then - return -1 -endi -if $data07 != 200 then - return -1 -endi -if $data08 != 200 then - return -1 -endi -if $data09 != 200 then - return -1 -endi - - - -$st = $stPrefix . avg -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - diff --git a/tests/script/general/stream/table_1.sim b/tests/script/general/stream/table_1.sim deleted file mode 100644 index f028b1626d156a65a3ff805f1479e9cb178a8d30..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/table_1.sim +++ /dev/null @@ -1,317 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = t1_db -$tbPrefix = t1_tb -$mtPrefix = t1_mt -$stPrefix = t1_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c1 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*) from $tb interval(1d) -print select count(*) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c1 -sql create table $st as select count(*) from $tb interval(1d) - -print =============== step3 c2 -sql select count(tbcol) from $tb interval(1d) -print select count(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c2 -sql create table $st as select count(tbcol) from $tb interval(1d) - -print =============== step4 c3 -sql select count(tbcol2) from $tb interval(1d) -print select count(tbcol2) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(tbcol2) from $tb interval(1d) - -print =============== step5 avg -sql select avg(tbcol) from $tb interval(1d) -print select avg(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 9.500000000 then - return -1 -endi - -$st = $stPrefix . av -sql create table $st as select avg(tbcol) from $tb interval(1d) - -print =============== step6 su -sql select sum(tbcol) from $tb interval(1d) -print select sum(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 190 then - return -1 -endi - -$st = $stPrefix . su -sql create table $st as select sum(tbcol) from $tb interval(1d) - -print =============== step7 mi -sql select min(tbcol) from $tb interval(1d) -print select min(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . mi -sql create table $st as select min(tbcol) from $tb interval(1d) - -print =============== step8 ma -sql select max(tbcol) from $tb interval(1d) -print select max(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . ma -sql create table $st as select max(tbcol) from $tb interval(1d) - -print =============== step9 fi -sql select first(tbcol) from $tb interval(1d) -print select first(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . fi -sql create table $st as select first(tbcol) from $tb interval(1d) - -print =============== step10 la -sql select last(tbcol) from $tb interval(1d) -print select last(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . la -sql create table $st as select last(tbcol) from $tb interval(1d) - -print =============== step11 st -sql select stddev(tbcol) from $tb interval(1d) -print select stddev(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 5.766281297 then - return -1 -endi - -$st = $stPrefix . std -sql create table $st as select stddev(tbcol) from $tb interval(1d) - -print =============== step12 le -sql select leastsquares(tbcol, 1, 1) from $tb interval(1d) -print select leastsquares(tbcol, 1, 1) from $tb interval(1d) ===> $data00 $data01 -#if $data01 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi - -$st = $stPrefix . le -sql create table $st as select leastsquares(tbcol, 1, 1) from $tb interval(1d) - -print =============== step13 -sql select top(tbcol, 1) from $tb interval(1d) - -print =============== step14 -sql select bottom(tbcol, 1) from $tb interval(1d) - -print =============== step15 pe - -sql select percentile(tbcol, 1) from $tb interval(1d) -print select percentile(tbcol, 1) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . pe -sql create table $st as select percentile(tbcol, 1) from $tb interval(1d) - -print =============== step16 -sql select diff(tbcol) from $tb interval(1d) -x step16 - return -1 -step16: - -print =============== step17 wh -sql select count(tbcol) from $tb where ts < now + 4m interval(1d) -print select count(tbcol) from $tb where ts < now + 4m interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . wh -#sql create table $st as select count(tbcol) from $tb where ts < now + 4m interval(1d) - -print =============== step18 as -sql select count(tbcol) from $tb interval(1d) -print select count(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . as -sql create table $st as select count(tbcol) as c from $tb interval(1d) - -print =============== step19 gb -sql select count(tbcol) from $tb interval(1d) group by tgcol -x step19 - return -1 -step19: - -print =============== step20 x -sql select count(tbcol) from $tb where ts < now + 4m interval(1d) group by tgcol -x step20 - return -1 -step20: - -print =============== step21 -print sleep 120 seconds -sleep 120000 - -print =============== step22 -$st = $stPrefix . c1 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c2 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . av -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 9.500000000 then - return -1 -endi - -$st = $stPrefix . su -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 190 then - return -1 -endi - -$st = $stPrefix . mi -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . ma -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . fi -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . la -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . std -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 5.766281297 then - return -1 -endi - -$st = $stPrefix . le -sql select * from $st -#print ===> select * from $st ===> $data00 $data01 -#if $data01 != @(0.000017, -25270086.331047)@ then -# return -1 -#endi - -$st = $stPrefix . pe -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0.190000000 then - return -1 -endi - -#$st = $stPrefix . wh -#sql select * from $st -#print ===> select * from $st ===> $data00 $data01 -#if $data01 != $rowNum then -# return -1 -#endi - -$st = $stPrefix . as -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi diff --git a/tests/script/general/stream/table_n.sim b/tests/script/general/stream/table_n.sim deleted file mode 100644 index a336772a9819ec54607265913eb65c26a2ec1a28..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/table_n.sim +++ /dev/null @@ -1,293 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = tn_db -$tbPrefix = tn_tb -$mtPrefix = tn_mt -$stPrefix = tn_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c3 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) -print select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) ===> $data00 $data01 $data02, $data03 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) - -print =============== step3 count32 -#total 32 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -# total 32 count in stream -$st = $stPrefix . c32 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -print =============== step4 count31 - -#total 31 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) -print ===> $data00 $data01 $data02, $data03 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -# total 31 count in stream will crash, 32 will error -$st = $stPrefix . c31 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -print =============== step5 avg ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $tb interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . avg -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $tb interval(1d) - -print =============== step6 others -sql select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $tb interval(1d) -print ===> $data01 $data02 $data03 -if $data01 != 5.766281297 then - return -1 -endi -#if $data02 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi -if $data03 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . ot -sql create table $st as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $tb interval(1d) - -print =============== step7 total ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 5.766281297 then - return -1 -endi -if $data08 != 0.190000000 then - return -1 -endi -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . to -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb interval(1d) - -print =============== step8 where -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . wh -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) - - -print =============== step9 as -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . as -#sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, stddev(tbcol) as a7, percentile(tbcol, 1) as a8, count(tbcol) as a9, leastsquares(tbcol, 1, 1) as a10 from $tb where ts < now + 4m interval(1d) - -print =============== step10 -print sleep 120 seconds -sleep 120000 - -print =============== step11 -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . c31 -sql select * from $st -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . avg -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . ot -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 5.766281297 then - return -1 -endi -#if $data02 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi -if $data03 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . to -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 5.766281297 then - return -1 -endi -if $data08 != 0.190000000 then - return -1 -endi -if $data09 != 20 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stream/testSuite.sim b/tests/script/general/stream/testSuite.sim index 44f886d02b2831bbd30e7d52a381cd6ea9b7ba27..4a9912b848f1976903199ad314352e70db0ca059 100644 --- a/tests/script/general/stream/testSuite.sim +++ b/tests/script/general/stream/testSuite.sim @@ -1,8 +1,5 @@ run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/general/table/delete_writing.sim b/tests/script/general/table/delete_writing.sim index b55d55eb8a6ef77f453f0d60152d6e1fb7a3269c..2a7fb09104391c0bf7dd04ed94ca6e710c3d9bb2 100644 --- a/tests/script/general/table/delete_writing.sim +++ b/tests/script/general/table/delete_writing.sim @@ -21,8 +21,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect sql create database db diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim index c6f7402b874115811e6422894c612c341ea5256f..3a8952391818bc02244f7455f694b6c1fe834288 100644 --- a/tests/script/general/wal/sync.sim +++ b/tests/script/general/wal/sync.sim @@ -31,7 +31,6 @@ system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032 print ============== deploy system sh/exec.sh -n dnode1 -s start -sleep 5001 sql connect sql create dnode $hostname2 @@ -43,8 +42,8 @@ print =============== step1 $x = 0 show1: $x = $x + 1 - sleep 2000 - if $x == 5 then + sleep 1000 + if $x == 10 then return -1 endi sql show mnodes -x show1 @@ -82,7 +81,7 @@ restful d1 table_rest 1591772800 30000 restful d1 table_rest 1591872800 30000 restful d1 table_rest 1591972800 30000 -sleep 1000 +sleep 100 sql select * from table_rest; print rows: $rows if $rows != 300000 then @@ -91,29 +90,51 @@ endi print =============== step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode1 -s start -x SIGINT -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi print =============== step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode2 -s start -x SIGINT -sleep 5000 +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a2 +endi print =============== step5 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 64a6c871fcf0ae610025b060105766d3b0cf3105..efe437fc1d319796dbf6179d570a38107618bc69 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -154,6 +154,7 @@ cd ../../../debug; make ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim +./test.sh -f general/parser/function.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim @@ -285,8 +286,12 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim ./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/lossdata.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim ./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim @@ -314,6 +319,7 @@ cd ../../../debug; make ./test.sh -f unique/mnode/mgmt24.sim ./test.sh -f unique/mnode/mgmt25.sim ./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt30.sim ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim @@ -325,16 +331,12 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -./test.sh -f general/parser/stream_on_sys.sim ./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim ./test.sh -f general/stream/metrics_replica1_vnoden.sim ./test.sh -f general/stream/restart_stream.sim ./test.sh -f general/stream/stream_3.sim ./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim ./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim ./test.sh -f general/stream/table_replica1_vnoden.sim ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt index 765e7139163952c43569e596d8ca8cf27d583cf6..9820a00f40b60263537f98cafb098b34b57032ff 100644 --- a/tests/script/jenkins/basic_1.txt +++ b/tests/script/jenkins/basic_1.txt @@ -150,6 +150,8 @@ ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim +./test.sh -f general/db/nosuchfile.sim +./test.sh -f general/parser/function.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim diff --git a/tests/script/jenkins/basic_2.txt b/tests/script/jenkins/basic_2.txt index 014313fafe9c7974baa7205cca41492dedee213a..4f5651219bf3fc8d87e9f95c73963ed16a2f28fd 100644 --- a/tests/script/jenkins/basic_2.txt +++ b/tests/script/jenkins/basic_2.txt @@ -26,6 +26,9 @@ cd ../../../debug; make ./test.sh -f general/tag/set.sim ./test.sh -f general/tag/smallint.sim ./test.sh -f general/tag/tinyint.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim ./test.sh -f general/user/authority.sim ./test.sh -f general/user/monitor.sim @@ -87,4 +90,15 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica2_repeat.sim ./test.sh -f unique/vnode/replica3_basic.sim ./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim \ No newline at end of file +./test.sh -f unique/vnode/replica3_vgroup.sim + +./test.sh -f unique/dnode/monitor.sim +./test.sh -f unique/dnode/monitor_bug.sim +./test.sh -f unique/dnode/simple.sim +./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt index 83b10a371cbc354acc079a6307fe0dbee27e3533..b44a2c6d44611da0360fe0a2c4266fd92f99aee2 100644 --- a/tests/script/jenkins/basic_3.txt +++ b/tests/script/jenkins/basic_3.txt @@ -1,38 +1,4 @@ -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -#./test.sh -f unique/mnode/mgmt25.sim -#./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - - - -./test.sh -f general/parser/stream_on_sys.sim -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -79,3 +45,31 @@ ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +./test.sh -f unique/stable/balance_replica1.sim +./test.sh -f unique/stable/dnode2_stop.sim +./test.sh -f unique/stable/dnode2.sim +./test.sh -f unique/stable/dnode3.sim +./test.sh -f unique/stable/replica2_dnode4.sim +./test.sh -f unique/stable/replica2_vnode3.sim +./test.sh -f unique/stable/replica3_dnode6.sim +./test.sh -f unique/stable/replica3_vnode3.sim + +./test.sh -f unique/mnode/mgmt20.sim +./test.sh -f unique/mnode/mgmt21.sim +./test.sh -f unique/mnode/mgmt22.sim +./test.sh -f unique/mnode/mgmt23.sim +./test.sh -f unique/mnode/mgmt24.sim +./test.sh -f unique/mnode/mgmt25.sim +./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt33.sim +./test.sh -f unique/mnode/mgmt34.sim +./test.sh -f unique/mnode/mgmtr2.sim + +./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim +./test.sh -f general/stream/restart_stream.sim +./test.sh -f general/stream/stream_3.sim +./test.sh -f general/stream/stream_restart.sim +./test.sh -f general/stream/table_del.sim +./test.sh -f general/stream/table_replica1_vnoden.sim diff --git a/tests/script/jenkins/simple.txt b/tests/script/jenkins/simple.txt index 2fe364bf26e0b3edf83278b085503e6ad2ac38f0..88ff4d660131bea5dee2f5ccd0dad9d81c7bed5f 100644 --- a/tests/script/jenkins/simple.txt +++ b/tests/script/jenkins/simple.txt @@ -147,9 +147,7 @@ cd ../../../debug; make ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim ./test.sh -f general/parser/bug.sim -#./test.sh -f general/parser/stream_on_sys.sim ./test.sh -f general/parser/repeatAlter.sim -#./test.sh -f general/parser/repeatStream.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index b271f5b726bd6e9e50265420ec6cfa6695b0f551..372bdc9d9d55635de6f36a0629b2f9c2aca5df07 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -10,6 +10,7 @@ cd ../../../debug; make ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim ./test.sh -f unique/cluster/cache.sim +./test.sh -f unique/cluster/vgroup100.sim ./test.sh -f unique/column/replica3.sim @@ -25,12 +26,21 @@ cd ../../../debug; make ./test.sh -f unique/db/replica_part.sim ./test.sh -f unique/dnode/alternativeRole.sim +./test.sh -f unique/dnode/monitor.sim +./test.sh -f unique/dnode/monitor_bug.sim +./test.sh -f unique/dnode/simple.sim ./test.sh -f unique/dnode/balance1.sim ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/lossdata.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim ./test.sh -f unique/dnode/vnode_clean.sim @@ -50,12 +60,14 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt20.sim ./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim ./test.sh -f unique/mnode/mgmt25.sim ./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt30.sim ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index ff1f9f53559c6da89ed2cdaf77c29f1859578ae6..e5e2194e87537f8496c9f6c5832d1a7d71842494 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -134,9 +134,6 @@ run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim run general/parser/repeatAlter.sim ##unsupport run general/parser/slimit_alter_tags.sim -##unsupport run general/parser/stream_on_sys.sim -run general/parser/stream.sim -#unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim run general/stable/metrics.sim @@ -214,14 +211,8 @@ run general/vector/table_mix.sim run general/vector/table_query.sim run general/vector/table_time.sim run general/stream/restart_stream.sim -run general/stream/stream_1.sim -run general/stream/stream_2.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/metrics_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index cd2f3772eb11a0c5d3a7a69fcb3d62162eadf14c..9b61a33d45f69a200b3f364f1553bdec5a3f52b5 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -132,10 +132,12 @@ echo "cqdebugFlag 143" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG +echo "slaveQuery 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "balanceInterval 1" >> $TAOS_CFG echo "clog 2" >> $TAOS_CFG #echo "cache 1" >> $TAOS_CFG echo "days 10" >> $TAOS_CFG diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index c647a35a5cc21c3a071b67b27330418a7b7ac57a..c9c85cb45dec7dd822b93916c452c274f2361ec9 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -360,12 +360,14 @@ print ============== step7: stop dnode3/dnode2, and cluster unable to provide se system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT sleep 3000 -sql_error select count(*) from $stb +sql select count(*) from $stb -x s71 +s71: print ============== step8: restart dnode2, and cluster Still unable to provide services system sh/exec.sh -n dnode2 -s start sleep 3000 -sql_error select count(*) from $stb +sql select count(*) from $stb -x s81 +s81: print ============== step9: restart dnode3, and cluster Resume service delivery system sh/exec.sh -n dnode3 -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 23ace47e3cf13cb18467c0482a30be97d5f03d33..387ae4625f392b8f108d7e9c34645905348d5e45 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -45,15 +45,30 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ============== step2: start dnode2 and add into cluster , then create database with replica 1, and create table, insert data system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 -$sleepTimer = 10000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi $db = db sql create database $db replica 1 @@ -84,7 +99,6 @@ while $i < $tblNum endw sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 @@ -92,118 +106,65 @@ endi print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep $sleepTimer - - -$loopCnt = 0 -wait_dnode2_offline_0: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline_0 -endi - system sh/exec.sh -n dnode2 -s start -sleep $sleepTimer +$x = 0 +a0: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -$loopCnt = 0 -wait_dnode2_reready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_reready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_reready +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a0 endi sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 endi - print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 3000 -sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode3_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode3_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode3_ready +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_3 != ready then + goto step2 endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode3_ready + +sql alter database $db replica 2 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -211,44 +172,22 @@ if $data00 != $totalRows then return -1 endi - print ============== step4: stop dnode2 for checking if sync ok system sh/exec.sh -n dnode2 -s stop -sleep $sleepTimer -$loopCnt = 0 -wait_dnode2_offline: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_offline -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline -endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode2_offline +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a2 endi -sleep $sleepTimer # waitting for move master vnode of dnode2 to dnode3 # check using select sql select count(*) from $stb print data00 $data00 @@ -258,38 +197,20 @@ endi print ============== step5: restart dnode2 system sh/exec.sh -n dnode2 -s start -sleep 3000 - -$loopCnt = 0 -wait_dnode2_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_ready +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -300,40 +221,39 @@ endi print ============== step6: start dnode4 and add into cluster , then alter replica from 2 to 3, and waiting sync system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 - -sql alter database $db replica 3 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode4_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes -if $rows != 4 then - sleep 2000 - goto wait_dnode4_ready +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_4 != ready then + goto step6 endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -#$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode4Status != ready then - sleep 2000 - goto wait_dnode4_ready + +sql alter database $db replica 3 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -343,44 +263,20 @@ endi print ============== step7: alter replica from 3 to 2, and waiting sync sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_2: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 $data10_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 $data10_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 $data10_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 $data10_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$thirdDnode_2 = $data8_1 -$thirdDnode_3 = $data8_2 -$thirdDnode_4 = $data8_3 -$thirdDnode_5 = $data8_4 - -if $thirdDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a5 endi -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -390,85 +286,20 @@ endi print ============== step8: alter replica from 2 to 1, and waiting sync sql alter database $db replica 1 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_1: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$sencodDnode_2 = $data6_1 -$sencodDnode_3 = $data6_2 -$sencodDnode_4 = $data6_3 -$sencodDnode_5 = $data6_4 - -if $sencodDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi - -$loopCnt = 0 -all_dnodes_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a6 endi -sql show dnodes -if $rows != 4 then - sleep 2000 - goto all_dnodes_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode1Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode2Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode3Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode4Status != ready then - sleep 2000 - goto all_dnodes_ready -endi - -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -481,39 +312,31 @@ sql drop dnode $hostname2 sql drop dnode $hostname3 sleep $sleepTimer -$loopCnt = 0 -wait_dnode23_dropped: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes if $rows != 2 then - sleep 2000 - goto wait_dnode23_dropped -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $dnode2Status != null then - sleep 2000 - goto wait_dnode23_dropped -endi -if $dnode3Status != null then - sleep 2000 - goto wait_dnode23_dropped + goto step9 endi -if $dnode4Status != ready then - return -1 + +$x = 0 +a7: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a7 endi sleep $sleepTimer #waiting move vnode from dnode3/dnode3 to dnode4 @@ -523,3 +346,13 @@ print data00 $data00 if $data00 != $totalRows then return -1 endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index c36e9f015e16512293b3113f9e46de0fd581c4ef..dd868e9eedbfc7bf81ab9667897221a9115015b1 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -140,3 +140,11 @@ if $data00 != $totalRows then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index 53209b0d5a07781ff8e22f406fc31823bdacc9b6..b3bd853ffc1f0953f4b021c051dfbbe70783aa6c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -158,13 +158,15 @@ if $dnode4Vtatus != offline then sleep 2000 goto wait_dnode4_vgroup_offline endi -if $dnode3Vtatus != unsynced then +if $dnode3Vtatus != master then sleep 2000 goto wait_dnode4_vgroup_offline endi -sql_error select count(*) from $stb -sql_error insert into $tb values (now, 9988) +sql select count(*) from $stb -x s31 +s31: +#sql_error insert into $tb values (now, 9988) -x s32 +#s32: print ============== step4: restart dnode2, then create database with replica 2, and create table, insert data system sh/exec.sh -n dnode2 -s start diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index ba973cbe06d92d6bd6a007dc5bd3cdb8fa6f2bfd..d4eb360efb2e1bd5687ed6e774736492c482bd3c 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -140,11 +140,9 @@ if $rows != 1 then goto wait_dnode4_vgroup_offline endi print show vgroups: -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data5_2 -$dnode3Vtatus = $data7_2 +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +$dnode4Vtatus = $data05 +$dnode3Vtatus = $data07 if $dnode4Vtatus != offline then sleep 2000 @@ -198,7 +196,3 @@ if $data00 != $totalRows then sleep 2000 goto wait_table_altered endi - - - - diff --git a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim index 3867aa3699cb84881c8b99eaa6ffa0813e8c785f..46d5e34fcbf8d70b4f678fc8e5abf311c9363ce8 100644 --- a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim +++ b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim @@ -14,6 +14,9 @@ sql alter table $stb add column f1 float $tblNum = $totalTableNum $alterTblNum = 10 +sql reset query cache +sleep 100 + $i = 1 while $i < $alterTblNum $tb = tb . $i diff --git a/tests/script/unique/big/balance.sim b/tests/script/unique/big/balance.sim index 91a7c538d27a30433aeea1fd1214e3c2d79283ec..f20939d458d512472cb822b44b5af0b378347017 100644 --- a/tests/script/unique/big/balance.sim +++ b/tests/script/unique/big/balance.sim @@ -21,7 +21,6 @@ system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 1000 print =============== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect $i = 0 @@ -92,8 +91,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show1: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -108,7 +107,7 @@ if $data2_2 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -143,7 +142,6 @@ endi print ========== step2 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 10000 print ========== step3 sql drop dnode $hostname2 @@ -151,8 +149,8 @@ sql drop dnode $hostname2 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -171,10 +169,9 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -212,8 +209,8 @@ sql drop dnode $hostname3 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -228,10 +225,9 @@ if $data2_3 != null then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -267,14 +263,31 @@ print ========== step5 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_4 != ready then + goto step5 +endi + sql alter database db replica 2 $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -289,7 +302,7 @@ if $data2_4 != 4 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -321,16 +334,14 @@ if $data00 != $totalNum then goto show5 endi - print ========== step6 -sleep 3000 sql alter database db replica 1 $x = 0 show6: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -345,7 +356,7 @@ if $data2_4 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum diff --git a/tests/script/unique/big/maxvnodes.sim b/tests/script/unique/big/maxvnodes.sim index 662d391e475f0b8afccbb37af5bae18fabe76621..eb17929ff49848acfe0744b4319aeb404a225adc 100644 --- a/tests/script/unique/big/maxvnodes.sim +++ b/tests/script/unique/big/maxvnodes.sim @@ -9,10 +9,11 @@ $totalRows = $totalVnodes * $maxTables system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v $totalVnodes +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c walLevel -v 2 system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v $totalVnodes - +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 print ========== prepare data system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/big/tcp.sim b/tests/script/unique/big/tcp.sim index 38cb667ec03b400e29cc1017f616434b650375d4..3c5cf92c7f183a84b2f1be0ce389bf86f482a42a 100644 --- a/tests/script/unique/big/tcp.sim +++ b/tests/script/unique/big/tcp.sim @@ -15,7 +15,6 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c debugFlag -v 131 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======================== dnode1 start @@ -52,7 +51,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step3 $i sql select count(*) from $mt print ===> $data00 $data01 @@ -83,7 +82,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step5 $i sql select count(*) from $mt where tgcol < 20200 print ===> $data00 $data01 diff --git a/tests/script/unique/cluster/balance1.sim b/tests/script/unique/cluster/balance1.sim index ec0c58d41651542e03d9f2ee710a38c5744c1f82..c98687a81ceae03d20fc205a121fcfc813c74be7 100644 --- a/tests/script/unique/cluster/balance1.sim +++ b/tests/script/unique/cluster/balance1.sim @@ -84,16 +84,14 @@ endi print ============================== step2 print ========= start dnode2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 5000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -129,13 +127,12 @@ sql insert into c_b1_t3 values(1520000024031, 31) print ============================== step4 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -164,16 +161,14 @@ print dnode4 ==> $dnode4Role print ============================== step5 print ========= add dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -212,13 +207,12 @@ sql_error create dnode $hostname1 print ============================== step8 sql drop dnode $hostname3 -sleep 15000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -253,13 +247,12 @@ endi print ============================== step9 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show9 @@ -302,8 +295,8 @@ sql insert into c_b1_t4 values(1520000024041, 41) $x = 0 show10: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show10 @@ -335,7 +328,23 @@ sql_error create table c_b1_t5 (t timestamp, i int) -x error3 print ============================== step13 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 +$x = 0 +step13: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode5 $data4_5 + +if $data4_5 != ready then + goto step13 +endi sql show mnodes $dnode1Role = $data2_1 @@ -371,19 +380,35 @@ $dnode5Vnodes = $data2_5 print dnode5 $dnode2Vnodes if $dnode1Vnodes != 2 then - return -1 + goto step13 endi if $dnode4Vnodes != 2 then - return -1 + goto step13 endi if $dnode5Vnodes != 2 then - return -1 + goto step13 endi print ============================== step14 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 15000 +$x = 0 +step14: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode4 $data4_5 + +if $data4_6 != ready then + goto step14 +endi sql create database c_b1_d7 sql use c_b1_d7 @@ -406,7 +431,7 @@ sql insert into c_b1_t8 values(1520000024081, 81) $x = 0 show14: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -440,7 +465,7 @@ print ============================== step17 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql use c_b1_d1 sql select * from c_b1_d1.c_b1_t1 diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index b8477965c6a1bb8a5fca0982ef25bbf993cf6ea2..ffd13445edbf76dbc96b82a14848ef6b788237fa 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -50,12 +50,48 @@ print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 4001 sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 4001 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi + sql create database c_b2_d1 replica 2 sql use c_b2_d1 @@ -112,13 +148,12 @@ endi print ============================== step2 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show2 @@ -155,13 +190,12 @@ print ============================== step3 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 10000 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show3 @@ -209,13 +243,12 @@ endi print ============================== step4 print ========= drop dnode3 sql drop dnode $hostname3 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -266,12 +299,11 @@ print ============================== step5 print ========= start dnode3 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -304,7 +336,7 @@ print dnode5 ==> $dnode5Role print ============================== step6 system sh/exec.sh -n dnode1 -s stop -x SIGINT print stop dnode1 and sleep 10000 -sleep 10000 +sleep 5000 sql drop dnode $hostname1 print drop dnode1 and sleep 9000 @@ -364,7 +396,7 @@ print ============================== step12 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql select * from c_b2_d1.c_b2_t1 order by t desc print $data01 $data11 $data21 $data31 $data41 diff --git a/tests/script/unique/cluster/balance3.sim b/tests/script/unique/cluster/balance3.sim index e3b8125d8c34a7e59aaac4f3df56e4650fb6cc68..c2e9a845149cd9df78b9528115c5108c1e6531d3 100644 --- a/tests/script/unique/cluster/balance3.sim +++ b/tests/script/unique/cluster/balance3.sim @@ -7,7 +7,6 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 system sh/deploy.sh -n dnode6 -i 6 system sh/deploy.sh -n dnode7 -i 7 -system sh/deploy.sh -n dnode8 -i 8 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 @@ -16,7 +15,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode6 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode7 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode8 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0 @@ -25,7 +23,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 0 -system sh/cfg.sh -n dnode8 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode1 -c wallevel -v 1 system sh/cfg.sh -n dnode2 -c wallevel -v 1 @@ -34,19 +31,52 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 1 system sh/cfg.sh -n dnode5 -c wallevel -v 1 system sh/cfg.sh -n dnode6 -c wallevel -v 1 system sh/cfg.sh -n dnode7 -c wallevel -v 1 -system sh/cfg.sh -n dnode8 -c wallevel -v 1 print ============== step1 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 2001 - sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3001 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi sql create database c_b3_d1 replica 3 sql use c_b3_d1 @@ -89,8 +119,6 @@ $dnode2Vnodes = $data2_2 print dnode2 $dnode2Vnodes $dnode3Vnodes = $data2_3 print dnode3 $dnode3Vnodes -$dnode4Vnodes = $data2_4 -print dnode4 $dnode4Vnodes if $dnode1Vnodes != 3 then goto show1 @@ -101,30 +129,22 @@ endi if $dnode3Vnodes != 3 then goto show1 endi -if $dnode4Vnodes != null then - goto show1 -endi sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step2 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -145,21 +165,16 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step3 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show3 @@ -189,25 +204,21 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 system sh/exec.sh -n dnode2 -s stop -x SIGINT print ============================== step4 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 10000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi + sql show dnodes -x show4 $dnode1Vnodes = $data2_1 print dnode1 $dnode1Vnodes @@ -228,8 +239,10 @@ print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 print dnode4 ==> $data2_4 print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 + +if $data2_4 != slave then + goto show4 +endi print ============================== step5 print ========= drop dnode3 @@ -239,8 +252,8 @@ sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -277,16 +290,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_5 != slave then + goto show5 +endi + print ============================== step6 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 9000 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show6 @@ -299,6 +315,15 @@ print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 print dnode6 $dnode6Vnodes +if $dnode1Vnodes != 2 then + goto show6 +endi +if $dnode4Vnodes != 2 then + goto show6 +endi +if $dnode5Vnodes != 3 then + goto show6 +endi if $dnode6Vnodes != 2 then goto show6 endi @@ -315,13 +340,12 @@ print dnode7 ==> $data2_7 print ============================== step7 print ========= drop dnode4 sql drop dnode $hostname4 -sleep 9000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show7 @@ -357,16 +381,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_6 != slave then + goto show7 +endi + print ============================== step8 sql create dnode $hostname7 system sh/exec.sh -n dnode7 -s start -sleep 9000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -379,6 +406,15 @@ print dnode6 $dnode6Vnodes $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes +if $dnode1Vnodes != 2 then + goto show8 +endi +if $dnode5Vnodes != 2 then + goto show8 +endi +if $dnode6Vnodes != 3 then + goto show8 +endi if $dnode7Vnodes != 2 then goto show8 endi @@ -393,19 +429,17 @@ print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 print ============================== step9 -print ========= drop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT -print stop dnode1 and sleep 10000 -sleep 10000 -sql drop dnode $hostname1 -print drop dnode1 and sleep 9000 -sleep 9000 +$x = 0 +show9: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show mnodes -x show9 -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 @@ -414,26 +448,27 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -if $dnode1Role != offline then - return -1 +if $data2_1 != offline then + goto show9 +endi +if $data2_5 != master then + goto show9 +endi +if $data2_6 != slave then + goto show9 endi -print ============================== step9.1 -sleep 2000 -system sh/exec.sh -n dnode1 -s start - +print ============================== step10 +sql drop dnode $hostname1 $x = 0 -show9: +show10: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi +sql show mnodes -x show10 -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 @@ -442,7 +477,31 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -sql show dnodes -x show9 +if $data2_1 != null then + goto show10 +endi +if $data2_5 != master then + goto show10 +endi +if $data2_6 != slave then + goto show10 +endi +if $data2_7 != slave then + goto show10 +endi + +print ============================== step11 +system sh/exec.sh -n dnode1 -s start + +$x = 0 +show11: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes -x show11 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -451,17 +510,16 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 3 then - goto show9 + goto show11 endi if $dnode6Vnodes != 3 then - goto show9 + goto show11 endi if $dnode7Vnodes != 3 then - goto show9 + goto show11 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql show mnodes print dnode1 ==> $data2_1 @@ -472,15 +530,13 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -print ============================== step11 -print ========= add db4 - +print ============================== step12 sql create database c_b3_d4 replica 3 sql use c_b3_d4 $x = 0 create4: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -491,16 +547,14 @@ sql insert into c_b3_t4 values(1520000022043, 43) sql insert into c_b3_t4 values(1520000023042, 42) sql insert into c_b3_t4 values(1520000024041, 41) -sleep 3000 - $x = 0 -show11: +show12: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi -sql show dnodes -x show11 +sql show dnodes -x show12 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -509,21 +563,18 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 4 then - goto show11 + goto show12 endi if $dnode6Vnodes != 4 then - goto show11 + goto show12 endi if $dnode7Vnodes != 4 then - goto show11 + goto show12 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 - print ============================== step13 sql reset query cache -sleep 1000 +sleep 200 print ========= check data @@ -590,4 +641,3 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/cluster/vgroup100.sim b/tests/script/unique/cluster/vgroup100.sim index 68a5bad6b32dc23c667900707d3e9027ac943077..656ed2ec44ebbb2d666ed2618e15d5ae34a5e525 100644 --- a/tests/script/unique/cluster/vgroup100.sim +++ b/tests/script/unique/cluster/vgroup100.sim @@ -17,7 +17,6 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 0 print ============================== step1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ============================== step2 @@ -27,8 +26,6 @@ system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 - $maxNum = 102 $maxNum = 12 @@ -92,13 +89,11 @@ print ============================== step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 print ============================== step6 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 10000 print ============================== step7 diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index fee47caadb7ab4cdab9db59b804a03eeb07a0149..f84339be79ad08951ff60c0df0d8992027c2ef98 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -18,7 +18,6 @@ system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 @@ -38,8 +37,6 @@ while $i < 2000 $i = $i + 1 endw -sleep 2500 - sql show db.vgroups if $rows != 2 then return -1 @@ -73,7 +70,6 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start @@ -92,7 +88,6 @@ if $data2_1 != master then goto step3 endi -sleep 1000 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index fd92c160d9fcc8f2b5f2e93e2d656c0f48a8e80e..682444e922a5e1403ac6046b85f1c48b0f4bc78a 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -31,11 +31,29 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + $loop = 0 begin: @@ -53,7 +71,6 @@ begin: sql insert into t13 values(now, 1 ) sql create table t14 (ts timestamp, i int) sql insert into t14 values(now, 1 ) - sleep 1200 sql create table t21 (ts timestamp, i int) sql insert into t21 values(now, 1 ) @@ -63,7 +80,6 @@ begin: sql insert into t23 values(now, 1 ) sql create table t24 (ts timestamp, i int) sql insert into t24 values(now, 1 ) - sleep 1200 sql create table t31 (ts timestamp, i int) sql insert into t31 values(now, 1 ) @@ -73,7 +89,6 @@ begin: sql insert into t33 values(now, 1 ) sql create table t34 (ts timestamp, i int) sql insert into t34 values(now, 1 ) - sleep 1200 sql create table t41 (ts timestamp, i int) sql insert into t41 values(now, 1 ) @@ -83,7 +98,6 @@ begin: sql insert into t43 values(now, 1 ) sql create table t44 (ts timestamp, i int) sql insert into t44 values(now, 1 ) - sleep 1200 sql create table t51 (ts timestamp, i int) sql insert into t51 values(now, 1 ) @@ -93,7 +107,6 @@ begin: sql insert into t53 values(now, 1 ) sql create table t54 (ts timestamp, i int) sql insert into t54 values(now, 1 ) - sleep 1200 sql create table t61 (ts timestamp, i int) sql insert into t61 values(now, 1 ) @@ -103,7 +116,6 @@ begin: sql insert into t63 values(now, 1 ) sql create table t64 (ts timestamp, i int) sql insert into t64 values(now, 1 ) - sleep 1200 sql create table t71 (ts timestamp, i int) sql insert into t71 values(now, 1 ) @@ -113,7 +125,6 @@ begin: sql insert into t73 values(now, 1 ) sql create table t74 (ts timestamp, i int) sql insert into t74 values(now, 1 ) - sleep 1200 sql create table t81 (ts timestamp, i int) sql insert into t81 values(now, 1 ) @@ -123,7 +134,6 @@ begin: sql insert into t83 values(now, 1 ) sql create table t84 (ts timestamp, i int) sql insert into t84 values(now, 1 ) - sleep 1200 sql show dnodes print dnode1 openVnodes $data2_1 @@ -138,29 +148,36 @@ begin: print ======== step2 $loop system sh/exec.sh -n dnode2 -s stop - sleep 1000 print ==> drop database $db sql drop database $db print ======== step3 $loop sleep 2000 system sh/exec.sh -n dnode2 -s start - sleep 15000 - sql show dnodes + + $x = 0 + step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + + sql show dnodes -x step3 print dnode1 openVnodes $data2_1 $data4_1 print dnode2 openVnodes $data2_2 $data4_2 if $data2_1 != 0 then - return -1 + goto step3 endi if $data2_2 != 0 then - return -1 + goto step3 endi if $data4_1 != ready then - return -1 + goto step3 endi if $data4_2 != ready then - return -1 + goto step3 endi print ===> test times : $loop @@ -171,7 +188,7 @@ begin: $loop = $loop + 1 sql reset query cache - sleep 1000 + sleep 100 goto begin diff --git a/tests/script/unique/db/replica_add12.sim b/tests/script/unique/db/replica_add12.sim index e29cf1ba73b471eaab1de058cf9a26752429aebb..d46187e4456acbd39c7b1f7689388af156d04f85 100644 --- a/tests/script/unique/db/replica_add12.sim +++ b/tests/script/unique/db/replica_add12.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -27,7 +32,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -65,8 +106,6 @@ if $rows != 1 then return -1 endi -sleep 2000 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -89,10 +128,40 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 10000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ======== step3 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -137,32 +206,54 @@ if $rows != 2 then endi sql reset query cache -sleep 2000 +sleep 200 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sql select * from d1.t1 -x s51 +s51: +#sql insert into d1.t1 values(now, 3) -x s52 +s52: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +print ========= step6 +system sh/exec.sh -n dnode2 -s start + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -print ===== insert data +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -print ========= step6 -system sh/exec.sh -n dnode2 -s start -sleep 5000 +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -sql insert into d1.t1 values(now, 3) -sql insert into d2.t2 values(now, 3) -sql insert into d3.t3 values(now, 3) -sql insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(now, 3) -x step6 +sql insert into d2.t2 values(now, 3) -x step6 +sql insert into d3.t3 values(now, 3) -x step6 +sql insert into d4.t4 values(now, 3) -x step6 sql select * from d1.t1 if $rows != 3 then @@ -185,28 +276,52 @@ if $rows != 3 then endi print ========= step61 - system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +#sql insert into d1.t1 values(now, 3) -x s61 +s61: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d2.t2 -x s62 +s62: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 -sql insert into d1.t1 values(now, 5) -sql insert into d2.t2 values(now, 5) -sql insert into d3.t3 values(now, 5) -sql insert into d4.t4 values(now, 5) +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql insert into d1.t1 values(now, 5) -x step7 +sql insert into d2.t2 values(now, 5) -x step7 +sql insert into d3.t3 values(now, 5) -x step7 +sql insert into d4.t4 values(now, 5) -x step7 sql select * from d1.t1 if $rows != 4 then @@ -231,8 +346,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add13.sim b/tests/script/unique/db/replica_add13.sim index 1df49ba6582b3e1502e9b19ac3b11626175a295e..13a5c9783228761c2b431f9bb1f571c29c05f080 100644 --- a/tests/script/unique/db/replica_add13.sim +++ b/tests/script/unique/db/replica_add13.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,41 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -72,17 +111,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 -print ======== step3 $x = 0 -show3: +a1: $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 + sleep 1000 + if $x == 40 then + return -1 endi - + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +print ======== step3 sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -133,9 +195,8 @@ endi print ========= step5 sql reset query cache -sleep 1000 +sleep 100 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000013, 3) sql insert into d2.t2 values(1589529000023, 3) @@ -164,9 +225,40 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000014, 4) sql insert into d2.t2 values(1589529000024, 4) @@ -199,9 +291,40 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000015, 5) sql insert into d2.t2 values(1589529000025, 5) @@ -230,9 +353,40 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000016, 6) sql insert into d2.t2 values(1589529000026, 6) @@ -262,8 +416,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add23.sim b/tests/script/unique/db/replica_add23.sim index 5da73cd117ef218d1421d4f7f6ee7026c9e6bef3..ac0bd6d9d7b09065c832fb8ad0698b97a9f94853 100644 --- a/tests/script/unique/db/replica_add23.sim +++ b/tests/script/unique/db/replica_add23.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,42 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -72,17 +112,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 -print ======== step3 $x = 0 -show3: +a1: $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 + sleep 1000 + if $x == 40 then + return -1 endi +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +print ======== step3 sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -132,12 +195,10 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 100 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql insert into d1.t1 values(1588262400003, 3) sql insert into d2.t2 values(1588262400003, 3) sql insert into d3.t3 values(1588262400003, 3) @@ -165,9 +226,40 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400004, 4) sql insert into d2.t2 values(1588262400004, 4) @@ -196,9 +288,40 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400005, 5) sql insert into d2.t2 values(1588262400005, 5) @@ -227,9 +350,40 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400006, 6) sql insert into d2.t2 values(1588262400006, 6) @@ -259,8 +413,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_part.sim b/tests/script/unique/db/replica_part.sim index de3f6203d2f9192d8f119145c6a6b8ac6cd674f8..9880ec666c8543676c406c843d14235a09ae13ee 100644 --- a/tests/script/unique/db/replica_part.sim +++ b/tests/script/unique/db/replica_part.sim @@ -14,6 +14,10 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +25,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -60,17 +95,79 @@ if $rows != 1 then endi print ========= step2 alter db -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========= step3 system sh/exec.sh -n dnode2 -s start -sleep 10000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi print ========= step4 sql insert into d1.t1 values(now, 2) @@ -100,38 +197,99 @@ endi print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 3) -x s1 s1: -sql insert into d2.t2 values(now, 3) -x s2 -s2: -sql insert into d3.t3 values(now, 3) -x s3 -s3: -sql insert into d4.t4 values(now, 3) -x s4 -s4: +#sql insert into d2.t2 values(now, 3) -x s2 +#s2: +#sql insert into d3.t3 values(now, 3) -x s3 +#s3: +#sql insert into d4.t4 values(now, 3) -x s4 +#s4: print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 -sql insert into d1.t1 values(now, 4) -x s5 -s5: +#sql insert into d1.t1 values(now, 4) -x s5 +#s5: sql insert into d2.t2 values(now, 4) -x s6 s6: -sql insert into d3.t3 values(now, 4) -x s7 -s7: -sql insert into d4.t4 values(now, 4) -x s8 -s8: +#sql insert into d3.t3 values(now, 4) -x s7 +#s7: +#sql insert into d4.t4 values(now, 4) -x s8 +#s8: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) @@ -145,9 +303,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce21.sim b/tests/script/unique/db/replica_reduce21.sim index a64f4370c89a162a0fd6cd8235eeebeee7e03806..d3a76485f81616497a8c4859aafefda7ef371c01 100644 --- a/tests/script/unique/db/replica_reduce21.sim +++ b/tests/script/unique/db/replica_reduce21.sim @@ -14,12 +14,42 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -36,6 +66,7 @@ sql insert into d2.t2 values(now, 1) sql insert into d1.t1 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) +sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -62,7 +93,21 @@ sql create database d5 replica 1 print ========= step3 alter d1 sql alter database d1 replica 1 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + print ========= step4 query d1 sql insert into d1.t1 values(now, 2) @@ -81,9 +126,9 @@ endi return print ========= step7 drop d1 sql drop database d1 -sleep 5000 - sql reset query cache +sleep 100 + print ========= step8 sql insert into d5.t5 values(now, 2) sql insert into d2.t2 values(now, 2) @@ -112,7 +157,6 @@ endi print ======== step9 stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d5.t5 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -140,10 +184,4 @@ if $rows != 3 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce31.sim b/tests/script/unique/db/replica_reduce31.sim index 3ed0a1e3b9b98dc1911142278e31ccfad25d1eb6..2313cbd85e05813a29bce15f96ac0094a3eb1664 100644 --- a/tests/script/unique/db/replica_reduce31.sim +++ b/tests/script/unique/db/replica_reduce31.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -63,21 +99,99 @@ print ========= step2 alter db sql_error alter database d1 replica 1 sql_error alter database d2 replica 1 sql_error alter database d3 replica 1 +sql_error alter database d4 replica 1 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 +sql alter database d4 replica 2 + +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi -sleep 8000 +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi sql alter database d1 replica 1 sql alter database d2 replica 1 sql alter database d3 replica 1 +sql alter database d4 replica 1 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi -sleep 8000 +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show dnodes +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 + +if $data02 != 0 then + goto a1 +endi +if $data12 != 2 then + goto a1 +endi +if $data22 != 2 then + goto a1 +endi print ========= step3 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) @@ -108,7 +222,39 @@ print ========= step4 alter db sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 -sleep 8000 +sql alter database d4 replica 2 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) @@ -137,10 +283,8 @@ endi print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 4) -x step1 step1: @@ -153,9 +297,7 @@ step4: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(now, 5) -x step5 step5: @@ -168,7 +310,6 @@ step8: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 sql insert into d1.t1 values(now, 6) sql insert into d2.t2 values(now, 6) @@ -183,8 +324,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce32.sim b/tests/script/unique/db/replica_reduce32.sim index 664a286be4f41284ddba42f3d053d9764e7a6408..a5eb78dacb6065413e06a50d7a0d9f7e28897d9f 100644 --- a/tests/script/unique/db/replica_reduce32.sim +++ b/tests/script/unique/db/replica_reduce32.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -64,7 +105,38 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ========= step3 sql insert into d1.t1 values(now, 2) @@ -93,45 +165,117 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 200 print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep 5000 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(now, 3) -x s41 +s41: +sql insert into d2.t2 values(now, 3) -x s42 +s42: +sql insert into d3.t3 values(now, 3) -x s43 +s43: +sql insert into d4.t4 values(now, 3) -x s44 +s44: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d1.t1 -x s45 +s45: +sql select * from d2.t2 -x s46 +s46: +sql select * from d3.t3 -x s47 +s47: +sql select * from d4.t4 -x s48 +s48: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +system sh/exec.sh -n dnode3 -s stop -x SIGINT sql reset query cache -sleep 1000 +sleep 100 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql_error insert into d1.t1 values(now, 3) -x s51 +s51: +sql_error insert into d2.t2 values(now, 3) -x s52 +s52: +sql_error insert into d3.t3 values(now, 3) -x s53 +s53: +sql_error insert into d4.t4 values(now, 3) -x s54 +s54: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) sql insert into d3.t3 values(now, 5) sql insert into d4.t4 values(now, 5) +sleep 1000 sql select * from d1.t1 print d1.t1 $rows @@ -147,9 +291,4 @@ print d4.t4 $rows system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/balance1.sim b/tests/script/unique/dnode/balance1.sim index 10d185f3536c1c7b4c61bc9401655d058ae17e88..8743204a03e06d0aefc66b2f70cbdc111e317237 100644 --- a/tests/script/unique/dnode/balance1.sim +++ b/tests/script/unique/dnode/balance1.sim @@ -33,7 +33,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 sql create database d1 sql create table d1.t1 (t timestamp, i int) @@ -50,15 +49,14 @@ if $data2_1 != 1 then endi print ========== step2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -124,8 +122,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -174,8 +172,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -258,7 +256,7 @@ endi system sh/exec.sh -n dnode3 -s stop -x SIGINT sql reset query cache -sleep 1000 +sleep 100 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance2.sim b/tests/script/unique/dnode/balance2.sim index 23897df690634759de027c9b26ad0449694242a9..7b07eb3f37334e78d1b2828120472d4ea9ebb5a1 100644 --- a/tests/script/unique/dnode/balance2.sim +++ b/tests/script/unique/dnode/balance2.sim @@ -32,7 +32,29 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database d1 replica 2 sql create table d1.t1 (t timestamp, i int) @@ -70,8 +92,8 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -90,7 +112,6 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step3 sql create dnode $hostname4 @@ -156,8 +177,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -189,8 +210,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -217,10 +238,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance3.sim b/tests/script/unique/dnode/balance3.sim index 9f7f3abb8bde66fdad63351d76e6d11e64db38aa..f5558d504e34608eafb2a076a8739024e13dcff2 100644 --- a/tests/script/unique/dnode/balance3.sim +++ b/tests/script/unique/dnode/balance3.sim @@ -38,7 +38,32 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi sql create database d1 replica 3 sql create table d1.t1 (t timestamp, i int) @@ -81,7 +106,7 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -106,7 +131,6 @@ if $data2_4 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start @@ -114,8 +138,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show3: $x = $x + 1 - sleep 4000 - if $x == 15 then + sleep 1000 + if $x == 60 then return -1 endi @@ -154,8 +178,8 @@ sql insert into d3.t3 values(now+5s, 31) $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -189,8 +213,8 @@ system sh/exec.sh -n dnode6 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -216,8 +240,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -245,10 +269,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/lossdata.sim b/tests/script/unique/dnode/lossdata.sim new file mode 100644 index 0000000000000000000000000000000000000000..89ba7169708eedb21c663a5e4fe0f897d42c5f43 --- /dev/null +++ b/tests/script/unique/dnode/lossdata.sim @@ -0,0 +1,165 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 2 +sql create table d1.t1 (t timestamp, i int) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 1 then + return -1 +endi +if $data2_3 != 1 then + return -1 +endi + +print ========== step3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi +if $data2_4 != 0 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step4 +sql drop dnode $hostname3 + +$i = 0 +$rowNum = 10000 + +while $i < $rowNum + $ts = 1500000000000 + $i + sql insert into d1.t1 values( $ts , $i ) + + $i = $i + 1 +endw + +print insert $rowNum finished + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 1 then + goto show4 +endi +if $data2_3 != null then + goto show4 +endi +if $data2_4 != 1 then + goto show4 +endi + +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print ========== step5 +sql select count(*) from d1.t1 +print select count(*) from d1.t1 ==> $data00 +if $data00 != $rowNum then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/m2.sim b/tests/script/unique/dnode/m2.sim new file mode 100644 index 0000000000000000000000000000000000000000..5fdf3b740081e74024f02616483af2c943d47eb9 --- /dev/null +++ b/tests/script/unique/dnode/m2.sim @@ -0,0 +1,367 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 2 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 15) +sql insert into d1.t1 values(now+2s, 14) +sql insert into d1.t1 values(now+3s, 13) +sql insert into d1.t1 values(now+4s, 12) +sql insert into d1.t1 values(now+5s, 11) + +sql create database d2 replica 2 +sql create table d2.t2 (t timestamp, i int) +sql insert into d2.t2 values(now+1s, 25) +sql insert into d2.t2 values(now+2s, 24) +sql insert into d2.t2 values(now+3s, 23) +sql insert into d2.t2 values(now+4s, 22) +sql insert into d2.t2 values(now+5s, 21) + +sql create database d3 replica 2 +sql create table d3.t3 (t timestamp, i int) +sql insert into d3.t3 values(now+1s, 35) +sql insert into d3.t3 values(now+2s, 34) +sql insert into d3.t3 values(now+3s, 33) +sql insert into d3.t3 values(now+4s, 32) +sql insert into d3.t3 values(now+5s, 31) + +sql create database d4 replica 2 +sql create table d4.t4 (t timestamp, i int) +sql insert into d4.t4 values(now+1s, 45) +sql insert into d4.t4 values(now+2s, 44) +sql insert into d4.t4 values(now+3s, 43) +sql insert into d4.t4 values(now+4s, 42) +sql insert into d4.t4 values(now+5s, 41) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 4 then + return -1 +endi +if $data2_3 != 4 then + return -1 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start +sql create dnode $hostname5 +system sh/exec.sh -n dnode5 -s start + + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 2 then + goto show3 +endi +if $data2_3 != 2 then + goto show3 +endi +if $data2_4 != 2 then + goto show3 +endi +if $data2_5 != 2 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +print ========== step4 +sql drop dnode $hostname2 + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql reset query cache +sleep 100 +system sh/exec.sh -n dnode2 -s stop -x SIGINT + +print ========== step5 +sql drop dnode $hostname3 + +$x = 0 +show5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show5 +endi +if $data2_3 != null then + goto show5 +endi +if $data2_4 != 4 then + goto show5 +endi +if $data2_5 != 4 then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql reset query cache +sleep 100 +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print ========== step6 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 11 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data41 != 15 then + return -1 +endi + +sql select * from d2.t2 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 21 then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data21 != 23 then + return -1 +endi +if $data31 != 24 then + return -1 +endi +if $data41 != 25 then + return -1 +endi + +sql select * from d3.t3 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +sql select * from d4.t4 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 41 then + return -1 +endi +if $data11 != 42 then + return -1 +endi +if $data21 != 43 then + return -1 +endi +if $data31 != 44 then + return -1 +endi +if $data41 != 45 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/m3.sim b/tests/script/unique/dnode/m3.sim new file mode 100644 index 0000000000000000000000000000000000000000..5850147d045d4e58fc8b700bcbb7d70318f0420a --- /dev/null +++ b/tests/script/unique/dnode/m3.sim @@ -0,0 +1,359 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 3 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 15) +sql insert into d1.t1 values(now+2s, 14) +sql insert into d1.t1 values(now+3s, 13) +sql insert into d1.t1 values(now+4s, 12) +sql insert into d1.t1 values(now+5s, 11) + +sql create database d2 replica 3 +sql create table d2.t2 (t timestamp, i int) +sql insert into d2.t2 values(now+1s, 25) +sql insert into d2.t2 values(now+2s, 24) +sql insert into d2.t2 values(now+3s, 23) +sql insert into d2.t2 values(now+4s, 22) +sql insert into d2.t2 values(now+5s, 21) + +sql create database d3 replica 3 +sql create table d3.t3 (t timestamp, i int) +sql insert into d3.t3 values(now+1s, 35) +sql insert into d3.t3 values(now+2s, 34) +sql insert into d3.t3 values(now+3s, 33) +sql insert into d3.t3 values(now+4s, 32) +sql insert into d3.t3 values(now+5s, 31) + +sql create database d4 replica 3 +sql create table d4.t4 (t timestamp, i int) +sql insert into d4.t4 values(now+1s, 45) +sql insert into d4.t4 values(now+2s, 44) +sql insert into d4.t4 values(now+3s, 43) +sql insert into d4.t4 values(now+4s, 42) +sql insert into d4.t4 values(now+5s, 41) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 4 then + return -1 +endi +if $data2_3 != 4 then + return -1 +endi +if $data2_4 != 4 then + return -1 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 3 then + return -1 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 2 then + return -1 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi + +print ========== step3 +sql create dnode $hostname5 +system sh/exec.sh -n dnode5 -s start + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 3 then + goto show3 +endi +if $data2_3 != 3 then + goto show3 +endi +if $data2_4 != 3 then + goto show3 +endi +if $data2_5 != 3 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 2 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +print ========== step4 +sql drop dnode $hostname2 + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show4 +endi +if $data2_3 != 4 then + goto show4 +endi +if $data2_4 != 4 then + goto show4 +endi +if $data2_5 != 4 then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 3 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql reset query cache +sleep 100 + +print ========== step5 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 11 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data41 != 15 then + return -1 +endi + +sql select * from d2.t2 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 21 then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data21 != 23 then + return -1 +endi +if $data31 != 24 then + return -1 +endi +if $data41 != 25 then + return -1 +endi + +sql select * from d3.t3 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +sql select * from d4.t4 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 41 then + return -1 +endi +if $data11 != 42 then + return -1 +endi +if $data21 != 43 then + return -1 +endi +if $data31 != 44 then + return -1 +endi +if $data41 != 45 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/offline3.sim b/tests/script/unique/dnode/offline3.sim new file mode 100644 index 0000000000000000000000000000000000000000..93c75e3b13333d55aea7cb2417413a14a1c13e62 --- /dev/null +++ b/tests/script/unique/dnode/offline3.sim @@ -0,0 +1,111 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode2 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode3 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 3 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 300 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +if $data2_1 != master then + goto step1 +endi + +print ========== step2 +sql create database db replica 3 +sql use db +sql create table tb (ts timestamp, value int) +sql insert into tb values (now, 1) +sql insert into tb values (now, 2) + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step2 +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi +if $data4_3 != ready then + goto step2 +endi +if $data4_4 != null then + goto step2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/reason.sim b/tests/script/unique/dnode/reason.sim index ad61a81b9782cf3d52c65059100c879423ae4843..c685b1129d924131d38e60369d97924250b3ad66 100644 --- a/tests/script/unique/dnode/reason.sim +++ b/tests/script/unique/dnode/reason.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode2 -i 2 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -18,38 +17,75 @@ endi print ========== step2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes -print dnode1 off: $data7_1 -print dnode2 off: $data7_2 +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -sleep 3000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_2 if $data7_2 != @status msg timeout@ then - return -1 + goto step3 endi print ========== step4 sql drop dnode $hostname2 -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes if $rows != 1 then - return -1 + goto step4 endi print ========== step5 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_3 if $data7_3 != @dnodeId not match@ then - return -1 + goto step5 endi print ========== step6 @@ -58,12 +94,19 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 3 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode4 off: $data7_4 if $data7_4 != @mnEqualVn not match@ then - return -1 + goto step6 endi print ========== step7 @@ -72,12 +115,19 @@ system sh/cfg.sh -n dnode5 -c statusInterval -v 3 system sh/exec.sh -n dnode5 -s start sql create dnode $hostname5 -sleep 3000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode5 off: $data7_5 if $data7_5 != @interval not match@ then - return -1 + goto step7 endi print ========== step8 @@ -86,12 +136,19 @@ system sh/cfg.sh -n dnode6 -c balance -v 0 system sh/exec.sh -n dnode6 -s start sql create dnode $hostname6 -sleep 3000 +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode6 off: $data7_6 if $data7_6 != @balance not match@ then - return -1 + goto step8 endi print ========== step9 @@ -100,12 +157,19 @@ system sh/cfg.sh -n dnode7 -c maxTablesPerVnode -v 3000 system sh/exec.sh -n dnode7 -s start sql create dnode $hostname7 -sleep 3000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode7 off: $data7_7 if $data7_7 != @maxTabPerVn not match@ then - return -1 + goto step9 endi print ========== step10 @@ -114,12 +178,19 @@ system sh/cfg.sh -n dnode8 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode8 -s start sql create dnode $hostname8 -sleep 3000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode8 off: $data7_8 if $data7_8 != @maxVgPerDb not match@ then - return -1 + goto step10 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/dnode/remove1.sim b/tests/script/unique/dnode/remove1.sim index 6bde68b8b5c7c5c7050141645fc689f08a5fe328..6f830d2cf8d50975a494854de9932fa74f41fb5c 100644 --- a/tests/script/unique/dnode/remove1.sim +++ b/tests/script/unique/dnode/remove1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,12 +99,11 @@ endi print ========== step3 sql drop dnode $hostname2 -sleep 7001 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -101,8 +118,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -115,7 +132,6 @@ if $data2_2 != null then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step5 sql create dnode $hostname4 @@ -124,8 +140,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 903e262be79a73e76b664c077b4faa0e90139262..f97e55164b14bc62237513a057cff21c36073179 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,7 +99,6 @@ endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT sql drop dnode $hostname2 -sleep 5000 sql show dnodes print dnode1 openVnodes $data2_1 @@ -91,14 +108,20 @@ print ========== step4 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 if $data2_3 != 0 then - return -1 + goto step4 endi print ============ step 4.1 @@ -107,8 +130,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/dnode/vnode_clean.sim b/tests/script/unique/dnode/vnode_clean.sim index f90f3d3fd010c7ac08305c3b38b12c7b48f5a70b..76311a6cacb2d2c06ff4125b689d3782c3e0dc98 100644 --- a/tests/script/unique/dnode/vnode_clean.sim +++ b/tests/script/unique/dnode/vnode_clean.sim @@ -45,8 +45,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -86,8 +86,8 @@ sql drop dnode $hostname2 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -106,15 +106,14 @@ endi system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========== step5 -sleep 5000 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes @@ -153,8 +152,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi @@ -184,8 +183,8 @@ sql insert into d4.t4 values(now+5s, 41) $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -208,8 +207,8 @@ sql drop dnode $hostname3 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -228,7 +227,6 @@ if $data2_4 != 4 then endi system sh/exec.sh -n dnode3 -s stop -sleep 5000 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/import/replica2.sim b/tests/script/unique/import/replica2.sim index f2836fe470562dd6229a9db7f694c7cfe7826cb3..54ce28543e741a1264e1bcf76c7152bee74564c0 100644 --- a/tests/script/unique/import/replica2.sim +++ b/tests/script/unique/import/replica2.sim @@ -27,11 +27,30 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi sql create database ir2db replica 2 days 7 sql use ir2db @@ -96,9 +115,22 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -163,9 +195,22 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -193,9 +238,20 @@ endi print ================= step13 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 +endi print ================= step14 #1520000000000 @@ -214,12 +270,24 @@ endi print ================= step15 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a4 +endi + +sql select * from tb; if $rows != 52 then - return -1 + goto a4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/import/replica3.sim b/tests/script/unique/import/replica3.sim index 3a9f03a7ea5140938b21b8f5c9dc983eb701c2c2..714141c412793f6608b11beb0a455ee3d37a3310 100644 --- a/tests/script/unique/import/replica3.sim +++ b/tests/script/unique/import/replica3.sim @@ -27,7 +27,6 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -35,6 +34,29 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database ir3db replica 3 days 7 sql use ir3db @@ -99,9 +121,20 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -166,9 +199,21 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step10 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -217,10 +262,21 @@ endi print ================= step15 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step15: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step15 +endi +sql select * from tb; if $rows != 52 then return -1 endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index 12f0013191b90cab8b85c9050f25ce182e1807b4..9b02933cbfc98c12c06ad1918d09dea9aa05ded0 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -194,36 +194,35 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 - -if $d2v2status != master then +if $data5_4 != master then + print $data5_4 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then +print $data5_2 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then +print $data5_3 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then + print $data7_4 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then + print $data7_3 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then + print $data7_2 sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -264,4 +263,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index 29da1fd77359146b6fb44fa8347e7b850947558d..90183949e7f516a5fb1c2127652217fefcc36d8d 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -196,36 +196,29 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 - -if $d2v2status != master then +if $data5_4 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -266,4 +259,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index 8a1132de2e62a6d97d37648435dd94d748a74c28..02e2cd02e1523df7ed0b1d2b29aade60585cd637 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -194,36 +194,29 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 - -if $d2v2status != master then +if $data5_4 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -264,4 +257,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt20.sim b/tests/script/unique/mnode/mgmt20.sim index e51d4299255248f3fb19371183536e5e62123655..ee9c2b914f237f619f342122a3b1fd2bccc5cf67 100644 --- a/tests/script/unique/mnode/mgmt20.sim +++ b/tests/script/unique/mnode/mgmt20.sim @@ -11,7 +11,6 @@ system sh/cfg.sh -n dnode2 -c monitor -v 1 print ============== step1 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 3000 sql connect print ============== step2 @@ -20,8 +19,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -51,6 +50,24 @@ $d1_first = $rows sql select * from log.dn2 $d2_first = $rows +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + goto show4 +endi +if $data2_2 != slave then + goto show4 +endi + sleep 3000 sql select * from log.dn1 $d1_second = $rows diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index d55e36d7fc627c3dcfb3e9c21c9396960a0c9287..1dc419b17dbd537e09833780c0104e304f8b71fe 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -9,7 +9,6 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -26,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -64,8 +63,8 @@ sql connect $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -82,13 +81,12 @@ endi print ============== step7 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 5000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/mnode/mgmt23.sim b/tests/script/unique/mnode/mgmt23.sim index d1820ef8c6a886fdbc55bda6d7b9e79a0ea51e1b..19c7b4ba762d4bf5a73c10c1afa39e927c7a1c91 100644 --- a/tests/script/unique/mnode/mgmt23.sim +++ b/tests/script/unique/mnode/mgmt23.sim @@ -25,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -65,7 +65,14 @@ endi print ============== step4 sql drop dnode $hostname2 -sleep 10000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -76,13 +83,13 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop @@ -93,7 +100,14 @@ sql create dnode $hostname2 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/exec.sh -n dnode2 -s start -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -104,19 +118,17 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 - sql_error show mnodes print ============== step7 @@ -126,7 +138,4 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt30.sim b/tests/script/unique/mnode/mgmt30.sim new file mode 100644 index 0000000000000000000000000000000000000000..a9488799334ef81c82e231bc7782993b7a33a4fa --- /dev/null +++ b/tests/script/unique/mnode/mgmt30.sim @@ -0,0 +1,68 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 3000 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 3000 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 3000 + +print ============== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +print dnode3 ==> $data3_3 +if $data2_1 != master then + return -1 +endi +if $data3_2 != null then + return -1 +endi +if $data3_3 != null then + return -1 +endi + +print ============== step2 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 5000 + +sql create dnode $hostname2 +sql create dnode $hostname3 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes +$dnode1Role = $data2_1 +$dnode2Role = $data2_2 +$dnode3Role = $data2_3 +print dnode1 ==> $dnode1Role +print dnode2 ==> $dnode2Role +print dnode3 ==> $dnode3Role + +if $dnode1Role != master then + goto step2 +endi +if $dnode2Role != slave then + goto step2 +endi +if $dnode3Role != slave then + goto step2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt33.sim b/tests/script/unique/mnode/mgmt33.sim index 205e1b05d376991d72d1676262ed50c4203d3bf3..ce7cdce35d8c0463564f46d26a0711d39340c8bf 100644 --- a/tests/script/unique/mnode/mgmt33.sim +++ b/tests/script/unique/mnode/mgmt33.sim @@ -28,7 +28,14 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -39,19 +46,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -62,18 +76,25 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi print ============== step4 sql drop dnode $hostname2 -sleep 8000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -84,25 +105,30 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step5 sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -113,20 +139,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != slave then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 - -sql show mnodes +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step6 $dnode1Role = $data2_1 $dnode2Role = $data2_4 $dnode3Role = $data2_3 @@ -135,7 +167,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != offline then - return -1 + goto step6 endi #if $dnode2Role != master then # return -1 @@ -146,9 +178,15 @@ endi print ============== step7 sql drop dnode $hostname1 -sleep 8000 - -sql show mnodes +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -157,7 +195,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != null then - return -1 + goto step7 endi #if $dnode2Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmt34.sim b/tests/script/unique/mnode/mgmt34.sim index 96e3133d762362cc47f656421dfab8470e37cb2e..d8a46b0955f59273279bbbc5c89c07c05db672d7 100644 --- a/tests/script/unique/mnode/mgmt34.sim +++ b/tests/script/unique/mnode/mgmt34.sim @@ -12,7 +12,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -32,7 +31,13 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -45,22 +50,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi if $dnode4Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -73,23 +85,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi if $dnode4Role != null then - return -1 + goto step3 endi print ============== step4 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 8000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -102,21 +120,27 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi if $dnode4Role != null then - return -1 + goto step4 endi print ============== step5 sql drop dnode $hostname2 -sleep 8000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -129,28 +153,32 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi if $dnode4Role != slave then - return -1 + goto step5 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step6 sql create dnode $hostname2 -sleep 8000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -163,23 +191,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step6 endi if $dnode2Role != null then - return -1 + goto step6 endi if $dnode3Role != slave then - return -1 + goto step6 endi if $dnode4Role != slave then - return -1 + goto step6 endi print ============== step7 system sh/exec.sh -n dnode1 -s stop -sleep 4000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -190,14 +224,19 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != offline then - return -1 + goto step7 endi print ============== step8 sql drop dnode $hostname1 -sleep 8000 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step8 $dnode1Role = $data2_1 $dnode2Role = $data2_5 $dnode3Role = $data2_3 @@ -208,10 +247,10 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != null then - return -1 + goto step8 endi if $dnode2Role != slave then - return -1 + goto step8 endi #if $dnode3Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmtr2.sim b/tests/script/unique/mnode/mgmtr2.sim index 1fba912d492785900f88686097074eaafc4445af..0c9f961d253e1ca57477f5612ed5f1f28c7b9e26 100644 --- a/tests/script/unique/mnode/mgmtr2.sim +++ b/tests/script/unique/mnode/mgmtr2.sim @@ -55,6 +55,14 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start print ============== step4 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + sql show mnodes $dnode1Role = $data2_1 @@ -65,20 +73,15 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != null then - return -1 + goto step4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/vnode/many.sim b/tests/script/unique/vnode/many.sim index 8bd6372ae9c677f5fc8475430ec3135b21409835..869c7f82953e99ef9e14a193f3894c4b53466b9e 100644 --- a/tests/script/unique/vnode/many.sim +++ b/tests/script/unique/vnode/many.sim @@ -23,7 +23,37 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db1 replica 2 diff --git a/tests/script/unique/vnode/replica2_repeat.sim b/tests/script/unique/vnode/replica2_repeat.sim index e862d745d4fac88137d1053b1bccd22d72c5f7cc..5dbb24437d3dc534018b788c5be1e29bf32cb72d 100644 --- a/tests/script/unique/vnode/replica2_repeat.sim +++ b/tests/script/unique/vnode/replica2_repeat.sim @@ -23,7 +23,38 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 2 @@ -64,7 +95,7 @@ print ======== step7 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_basic.sim b/tests/script/unique/vnode/replica3_basic.sim index ab5bebb50daf8a3be63d039152a020f0fff7000b..0ff42b523b8982c85bd84bb251715585a66137fc 100644 --- a/tests/script/unique/vnode/replica3_basic.sim +++ b/tests/script/unique/vnode/replica3_basic.sim @@ -16,14 +16,48 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi $N = 10 $table = table_r3 $db = db1 -sleep 3000 - print =================== step 1 sql create database $db replica 3 @@ -66,7 +100,21 @@ endi print =================== step 4 system sh/exec.sh -n dnode2 -s start -sleep 2000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step4 +endi + $y = $x + $N $expect = $N * 3 while $x < $y @@ -83,7 +131,6 @@ endi print =================== step 5 system sh/exec.sh -n dnode3 -s stop -sleep 2000 $y = $x + $N $expect = $N * 4 while $x < $y @@ -100,7 +147,21 @@ endi print =================== step 6 system sh/exec.sh -n dnode3 -s start -sleep 2000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + $y = $x + $N $expect = $N * 5 while $x < $y @@ -117,7 +178,6 @@ endi print =================== step 7 system sh/exec.sh -n dnode1 -s stop -sleep 2000 $y = $x + $N $expect = $N * 6 while $x < $y @@ -134,7 +194,21 @@ endi print =================== step 8 system sh/exec.sh -n dnode1 -s start -sleep 2000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + $y = $x + $N $expect = $N * 7 while $x < $y diff --git a/tests/script/unique/vnode/replica3_repeat.sim b/tests/script/unique/vnode/replica3_repeat.sim index 4b5c852de8a6f841ae75b9716577ae735f970b3f..d866fb05d89b5441d83390a154324fb4290c451d 100644 --- a/tests/script/unique/vnode/replica3_repeat.sim +++ b/tests/script/unique/vnode/replica3_repeat.sim @@ -26,7 +26,42 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 3 @@ -75,7 +110,7 @@ print ======== step8 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/test-all.sh b/tests/test-all.sh index 14b649eddfe92e8c557f81d27a0d1b95b35602c3..19d78032554682d533c89bc4e4bd473689364d87 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -6,7 +6,16 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' - +function git_branch { + branch="`git branch 2>/dev/null | grep "^\*" | sed -e "s/^\*\ //"`" + if [ "${branch}" != "" ];then + if [ "${branch}" = "(no branch)" ];then + branch="(`git rev-parse --short HEAD`...)" + fi + branch=(${branch////_}) + echo "$branch" + fi +} function runSimCaseOneByOne { while read -r line; do if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then @@ -25,6 +34,37 @@ function runSimCaseOneByOne { fi done < $1 } +function runSimCaseOneByOnefq { + while read -r line; do + if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then + case=`echo $line | grep sim$ |awk '{print $NF}'` + + start_time=`date +%s` + IN_TDINTERNAL="community" + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ + ( grep 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log + else + ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ + ( grep 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log + fi + + out_log=`tail -1 out.log ` + if [[ $out_log =~ 'failed' ]];then + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cp -r ../../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S"` + else + cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + exit 8 + fi + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log + fi + done < $1 +} function runPyCaseOneByOne { while read -r line; do @@ -52,7 +92,33 @@ function runPyCaseOneByOne { fi done < $1 } - +function runPyCaseOneByOnefq { + while read -r line; do + if [[ $line =~ ^python.* ]]; then + if [[ $line != *sleep* ]]; then + + if [[ $line =~ '-r' ]];then + case=`echo $line|awk '{print $4}'` + else + case=`echo $line|awk '{print $NF}'` + fi + start_time=`date +%s` + $line > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ + echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log + end_time=`date +%s` + out_log=`tail -1 pytest-out.log ` + if [[ $out_log =~ 'failed' ]];then + cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` + exit 8 + fi + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log + else + $line > /dev/null 2>&1 + fi + fi + done < $1 +} totalFailed=0 totalPyFailed=0 @@ -78,6 +144,15 @@ if [ "$2" != "python" ]; then elif [ "$1" == "b3" ]; then echo "### run TSIM b3 test ###" runSimCaseOneByOne jenkins/basic_3.txt + elif [ "$1" == "b1fq" ]; then + echo "### run TSIM b1 test ###" + runSimCaseOneByOnefq jenkins/basic_1.txt + elif [ "$1" == "b2fq" ]; then + echo "### run TSIM b2 test ###" + runSimCaseOneByOnefq jenkins/basic_2.txt + elif [ "$1" == "b3fq" ]; then + echo "### run TSIM b3 test ###" + runSimCaseOneByOnefq jenkins/basic_3.txt elif [ "$1" == "smoke" ] || [ -z "$1" ]; then echo "### run TSIM smoke test ###" runSimCaseOneByOne basicSuite.sim @@ -137,12 +212,15 @@ if [ "$2" != "sim" ]; then elif [ "$1" == "pytest" ]; then echo "### run Python full test ###" runPyCaseOneByOne fulltest.sh + elif [ "$1" == "pytestfq" ]; then + echo "### run Python full test ###" + runPyCaseOneByOnefq fulltest.sh elif [ "$1" == "p1" ]; then echo "### run Python_1 test ###" - runPyCaseOneByOne pytest_1.sh + runPyCaseOneByOnefq pytest_1.sh elif [ "$1" == "p2" ]; then echo "### run Python_2 test ###" - runPyCaseOneByOne pytest_2.sh + runPyCaseOneByOnefq pytest_2.sh elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then exit $(($totalFailed + $totalPyFailed)) elif [ "$1" == "smoke" ] || [ -z "$1" ]; then diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 11480a8ba2f7c2a78b8628e3edabe6e0eb122410..2eb8ee1614b286f3827705865cf073a7eded0c88 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -31,8 +31,8 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - add_executable(createNormalTable createNormalTable.c) - target_link_libraries(createNormalTable taos_static tutil common pthread) + #add_executable(createNormalTable createNormalTable.c) + #target_link_libraries(createNormalTable taos_static tutil common pthread) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) @@ -46,7 +46,7 @@ IF (TD_LINUX) #add_executable(invalidTableId invalidTableId.c) #target_link_libraries(invalidTableId taos_static tutil common pthread) - add_executable(hashIterator hashIterator.c) - target_link_libraries(hashIterator taos_static tutil common pthread) + #add_executable(hashIterator hashIterator.c) + #target_link_libraries(hashIterator taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/createTablePerformance.c b/tests/test/c/createTablePerformance.c index eae104291a3bfcbb37e6a2c89d61214c61d4b6c0..b94c687f2cba7310949b0a3b12b6f4fc007e5a9a 100644 --- a/tests/test/c/createTablePerformance.c +++ b/tests/test/c/createTablePerformance.c @@ -32,6 +32,7 @@ int32_t numOfThreads = 30; int32_t numOfTables = 100000; int32_t replica = 1; int32_t numOfColumns = 2; +TAOS * con = NULL; typedef struct { int32_t tableBeginIndex; @@ -84,13 +85,14 @@ int main(int argc, char *argv[]) { pthread_attr_destroy(&thattr); free(pInfo); + taos_close(con); } void createDbAndSTable() { pPrint("start to create db and stable"); char qstr[64000]; - TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { pError("failed to connect to DB, reason:%s", taos_errstr(con)); exit(1); @@ -127,8 +129,6 @@ void createDbAndSTable() { exit(0); } taos_free_result(pSql); - - taos_close(con); } void *threadFunc(void *param) { @@ -136,12 +136,6 @@ void *threadFunc(void *param) { char qstr[65000]; int code; - TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); - if (con == NULL) { - pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); - exit(1); - } - sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pSql = taos_query(con, qstr); taos_free_result(pSql); @@ -170,7 +164,6 @@ void *threadFunc(void *param) { pInfo->createTableSpeed = speed; pPrint("thread:%d, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, seconds, speed); - taos_close(con); return 0; }