diff --git a/.travis.yml b/.travis.yml index eb69370418a9c83c7b8bfe5daa1d6ead19150243..f6a3900f7a87c1af58b757590ecc07f3bfe9ce53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -146,7 +146,7 @@ matrix: branch_pattern: coverity_scan - os: linux - dist: trusty + dist: xenial language: c git: - depth: 1 @@ -157,7 +157,7 @@ matrix: - build-essential - cmake env: - - DESC="trusty/gcc-4.8 build" + - DESC="xenial build" before_script: - export TZ=Asia/Harbin @@ -227,7 +227,7 @@ matrix: - os: linux arch: arm64 - dist: trusty + dist: xenial language: c git: - depth: 1 @@ -238,7 +238,7 @@ matrix: - build-essential - cmake env: - - DESC="trusty/gcc-4.8 build" + - DESC="xenial build" before_script: - export TZ=Asia/Harbin diff --git a/Jenkinsfile b/Jenkinsfile index 583fcc7efdacfc6aebe67b5c2233bb1b42efbb48..9544343bec6ef964fb15cf94c7a1a7c93d98810f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,12 +41,15 @@ def pre_test(){ cd ${WKC} git checkout develop + git reset --hard HEAD~10 git pull git fetch git checkout ${CHANGE_BRANCH} + git reset --hard HEAD~10 + git pull git merge develop cd ${WK} - git reset --hard + git reset --hard HEAD~10 git checkout develop git pull cd ${WK} @@ -79,14 +82,27 @@ pipeline { changeRequest() } parallel { - stage('python') { - agent{label 'pytest'} + stage('python_1') { + agent{label 'p1'} + steps { + + pre_test() + sh ''' + cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh p1 + date''' + } + } + stage('python_2') { + agent{label 'p2'} steps { pre_test() sh ''' cd ${WKC}/tests - ./test-all.sh pytestfq + find pytest -name '*'sql|xargs rm -rf + ./test-all.sh p2 date''' } } diff --git a/alert/go.mod b/alert/go.mod index 01c557d5646714869ad263a9c8fe2fa262fe9a43..43920f1f1aea2a95d02fbaddeab8ebc80ad1e81f 100644 --- a/alert/go.mod +++ b/alert/go.mod @@ -5,7 +5,7 @@ go 1.14 require ( github.com/jmoiron/sqlx v1.2.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible - github.com/taosdata/driver-go v0.0.0-20200727182616-1a3b1941c206 + github.com/taosdata/driver-go v0.0.0-20201113094317-050667e5b4d0 go.uber.org/zap v1.14.1 google.golang.org/appengine v1.6.5 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c diff --git a/cmake/version.inc b/cmake/version.inc index e11e782a782ebccf4c567d0d711dc472edda25f7..49f01d00bc18ef4f0e1d87ac06b4b1bf09784268 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.11.0") + SET(TD_VER_NUMBER "2.0.13.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/deps/libcurl/include/curl/curl.h b/deps/libcurl/include/curl/curl.h new file mode 100644 index 0000000000000000000000000000000000000000..84229bb698b77a5ed5d67057772ed23188e191e8 --- /dev/null +++ b/deps/libcurl/include/curl/curl.h @@ -0,0 +1,2415 @@ +#ifndef __CURL_CURL_H +#define __CURL_CURL_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* + * If you have libcurl problems, all docs and details are found here: + * http://curl.haxx.se/libcurl/ + * + * curl-library mailing list subscription and unsubscription web interface: + * http://cool.haxx.se/mailman/listinfo/curl-library/ + */ + +#include "curlver.h" /* libcurl version defines */ +#include "curlbuild.h" /* libcurl build definitions */ +#include "curlrules.h" /* libcurl rules enforcement */ + +/* + * Define WIN32 when build target is Win32 API + */ + +#if (defined(_WIN32) || defined(__WIN32__)) && \ + !defined(WIN32) && !defined(__SYMBIAN32__) +#define WIN32 +#endif + +#include +#include + +#if defined(__FreeBSD__) && (__FreeBSD__ >= 2) +/* Needed for __FreeBSD_version symbol definition */ +#include +#endif + +/* The include stuff here below is mainly for time_t! */ +#include +#include + +#if defined(WIN32) && !defined(_WIN32_WCE) && !defined(__CYGWIN__) +#if !(defined(_WINSOCKAPI_) || defined(_WINSOCK_H) || \ + defined(__LWIP_OPT_H__) || defined(LWIP_HDR_OPT_H)) +/* The check above prevents the winsock2 inclusion if winsock.h already was + included, since they can't co-exist without problems */ +#include +#include +#endif +#endif + +/* HP-UX systems version 9, 10 and 11 lack sys/select.h and so does oldish + libc5-based Linux systems. Only include it on systems that are known to + require it! */ +#if defined(_AIX) || defined(__NOVELL_LIBC__) || defined(__NetBSD__) || \ + defined(__minix) || defined(__SYMBIAN32__) || defined(__INTEGRITY) || \ + defined(ANDROID) || defined(__ANDROID__) || defined(__OpenBSD__) || \ + (defined(__FreeBSD_version) && (__FreeBSD_version < 800000)) +#include +#endif + +#if !defined(WIN32) && !defined(_WIN32_WCE) +#include +#endif + +#if !defined(WIN32) && !defined(__WATCOMC__) && !defined(__VXWORKS__) +#include +#endif + +#ifdef __BEOS__ +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void CURL; + +/* + * libcurl external API function linkage decorations. + */ + +#ifdef CURL_STATICLIB +# define CURL_EXTERN +#elif defined(WIN32) || defined(_WIN32) || defined(__SYMBIAN32__) +# if defined(BUILDING_LIBCURL) +# define CURL_EXTERN __declspec(dllexport) +# else +# define CURL_EXTERN __declspec(dllimport) +# endif +#elif defined(BUILDING_LIBCURL) && defined(CURL_HIDDEN_SYMBOLS) +# define CURL_EXTERN CURL_EXTERN_SYMBOL +#else +# define CURL_EXTERN +#endif + +#ifndef curl_socket_typedef +/* socket typedef */ +#if defined(WIN32) && !defined(__LWIP_OPT_H__) && !defined(LWIP_HDR_OPT_H) +typedef SOCKET curl_socket_t; +#define CURL_SOCKET_BAD INVALID_SOCKET +#else +typedef int curl_socket_t; +#define CURL_SOCKET_BAD -1 +#endif +#define curl_socket_typedef +#endif /* curl_socket_typedef */ + +struct curl_httppost { + struct curl_httppost *next; /* next entry in the list */ + char *name; /* pointer to allocated name */ + long namelength; /* length of name length */ + char *contents; /* pointer to allocated data contents */ + long contentslength; /* length of contents field, see also + CURL_HTTPPOST_LARGE */ + char *buffer; /* pointer to allocated buffer contents */ + long bufferlength; /* length of buffer field */ + char *contenttype; /* Content-Type */ + struct curl_slist* contentheader; /* list of extra headers for this form */ + struct curl_httppost *more; /* if one field name has more than one + file, this link should link to following + files */ + long flags; /* as defined below */ + +/* specified content is a file name */ +#define CURL_HTTPPOST_FILENAME (1<<0) +/* specified content is a file name */ +#define CURL_HTTPPOST_READFILE (1<<1) +/* name is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRNAME (1<<2) +/* contents is only stored pointer do not free in formfree */ +#define CURL_HTTPPOST_PTRCONTENTS (1<<3) +/* upload file from buffer */ +#define CURL_HTTPPOST_BUFFER (1<<4) +/* upload file from pointer contents */ +#define CURL_HTTPPOST_PTRBUFFER (1<<5) +/* upload file contents by using the regular read callback to get the data and + pass the given pointer as custom pointer */ +#define CURL_HTTPPOST_CALLBACK (1<<6) +/* use size in 'contentlen', added in 7.46.0 */ +#define CURL_HTTPPOST_LARGE (1<<7) + + char *showfilename; /* The file name to show. If not set, the + actual file name will be used (if this + is a file part) */ + void *userp; /* custom pointer used for + HTTPPOST_CALLBACK posts */ + curl_off_t contentlen; /* alternative length of contents + field. Used if CURL_HTTPPOST_LARGE is + set. Added in 7.46.0 */ +}; + +/* This is the CURLOPT_PROGRESSFUNCTION callback proto. It is now considered + deprecated but was the only choice up until 7.31.0 */ +typedef int (*curl_progress_callback)(void *clientp, + double dltotal, + double dlnow, + double ultotal, + double ulnow); + +/* This is the CURLOPT_XFERINFOFUNCTION callback proto. It was introduced in + 7.32.0, it avoids floating point and provides more detailed information. */ +typedef int (*curl_xferinfo_callback)(void *clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow); + +#ifndef CURL_MAX_WRITE_SIZE + /* Tests have proven that 20K is a very bad buffer size for uploads on + Windows, while 16K for some odd reason performed a lot better. + We do the ifndef check to allow this value to easier be changed at build + time for those who feel adventurous. The practical minimum is about + 400 bytes since libcurl uses a buffer of this size as a scratch area + (unrelated to network send operations). */ +#define CURL_MAX_WRITE_SIZE 16384 +#endif + +#ifndef CURL_MAX_HTTP_HEADER +/* The only reason to have a max limit for this is to avoid the risk of a bad + server feeding libcurl with a never-ending header that will cause reallocs + infinitely */ +#define CURL_MAX_HTTP_HEADER (100*1024) +#endif + +/* This is a magic return code for the write callback that, when returned, + will signal libcurl to pause receiving on the current transfer. */ +#define CURL_WRITEFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_write_callback)(char *buffer, + size_t size, + size_t nitems, + void *outstream); + + + +/* enumeration of file types */ +typedef enum { + CURLFILETYPE_FILE = 0, + CURLFILETYPE_DIRECTORY, + CURLFILETYPE_SYMLINK, + CURLFILETYPE_DEVICE_BLOCK, + CURLFILETYPE_DEVICE_CHAR, + CURLFILETYPE_NAMEDPIPE, + CURLFILETYPE_SOCKET, + CURLFILETYPE_DOOR, /* is possible only on Sun Solaris now */ + + CURLFILETYPE_UNKNOWN /* should never occur */ +} curlfiletype; + +#define CURLFINFOFLAG_KNOWN_FILENAME (1<<0) +#define CURLFINFOFLAG_KNOWN_FILETYPE (1<<1) +#define CURLFINFOFLAG_KNOWN_TIME (1<<2) +#define CURLFINFOFLAG_KNOWN_PERM (1<<3) +#define CURLFINFOFLAG_KNOWN_UID (1<<4) +#define CURLFINFOFLAG_KNOWN_GID (1<<5) +#define CURLFINFOFLAG_KNOWN_SIZE (1<<6) +#define CURLFINFOFLAG_KNOWN_HLINKCOUNT (1<<7) + +/* Content of this structure depends on information which is known and is + achievable (e.g. by FTP LIST parsing). Please see the url_easy_setopt(3) man + page for callbacks returning this structure -- some fields are mandatory, + some others are optional. The FLAG field has special meaning. */ +struct curl_fileinfo { + char *filename; + curlfiletype filetype; + time_t time; + unsigned int perm; + int uid; + int gid; + curl_off_t size; + long int hardlinks; + + struct { + /* If some of these fields is not NULL, it is a pointer to b_data. */ + char *time; + char *perm; + char *user; + char *group; + char *target; /* pointer to the target filename of a symlink */ + } strings; + + unsigned int flags; + + /* used internally */ + char * b_data; + size_t b_size; + size_t b_used; +}; + +/* return codes for CURLOPT_CHUNK_BGN_FUNCTION */ +#define CURL_CHUNK_BGN_FUNC_OK 0 +#define CURL_CHUNK_BGN_FUNC_FAIL 1 /* tell the lib to end the task */ +#define CURL_CHUNK_BGN_FUNC_SKIP 2 /* skip this chunk over */ + +/* if splitting of data transfer is enabled, this callback is called before + download of an individual chunk started. Note that parameter "remains" works + only for FTP wildcard downloading (for now), otherwise is not used */ +typedef long (*curl_chunk_bgn_callback)(const void *transfer_info, + void *ptr, + int remains); + +/* return codes for CURLOPT_CHUNK_END_FUNCTION */ +#define CURL_CHUNK_END_FUNC_OK 0 +#define CURL_CHUNK_END_FUNC_FAIL 1 /* tell the lib to end the task */ + +/* If splitting of data transfer is enabled this callback is called after + download of an individual chunk finished. + Note! After this callback was set then it have to be called FOR ALL chunks. + Even if downloading of this chunk was skipped in CHUNK_BGN_FUNC. + This is the reason why we don't need "transfer_info" parameter in this + callback and we are not interested in "remains" parameter too. */ +typedef long (*curl_chunk_end_callback)(void *ptr); + +/* return codes for FNMATCHFUNCTION */ +#define CURL_FNMATCHFUNC_MATCH 0 /* string corresponds to the pattern */ +#define CURL_FNMATCHFUNC_NOMATCH 1 /* pattern doesn't match the string */ +#define CURL_FNMATCHFUNC_FAIL 2 /* an error occurred */ + +/* callback type for wildcard downloading pattern matching. If the + string matches the pattern, return CURL_FNMATCHFUNC_MATCH value, etc. */ +typedef int (*curl_fnmatch_callback)(void *ptr, + const char *pattern, + const char *string); + +/* These are the return codes for the seek callbacks */ +#define CURL_SEEKFUNC_OK 0 +#define CURL_SEEKFUNC_FAIL 1 /* fail the entire transfer */ +#define CURL_SEEKFUNC_CANTSEEK 2 /* tell libcurl seeking can't be done, so + libcurl might try other means instead */ +typedef int (*curl_seek_callback)(void *instream, + curl_off_t offset, + int origin); /* 'whence' */ + +/* This is a return code for the read callback that, when returned, will + signal libcurl to immediately abort the current transfer. */ +#define CURL_READFUNC_ABORT 0x10000000 +/* This is a return code for the read callback that, when returned, will + signal libcurl to pause sending data on the current transfer. */ +#define CURL_READFUNC_PAUSE 0x10000001 + +typedef size_t (*curl_read_callback)(char *buffer, + size_t size, + size_t nitems, + void *instream); + +typedef enum { + CURLSOCKTYPE_IPCXN, /* socket created for a specific IP connection */ + CURLSOCKTYPE_ACCEPT, /* socket created by accept() call */ + CURLSOCKTYPE_LAST /* never use */ +} curlsocktype; + +/* The return code from the sockopt_callback can signal information back + to libcurl: */ +#define CURL_SOCKOPT_OK 0 +#define CURL_SOCKOPT_ERROR 1 /* causes libcurl to abort and return + CURLE_ABORTED_BY_CALLBACK */ +#define CURL_SOCKOPT_ALREADY_CONNECTED 2 + +typedef int (*curl_sockopt_callback)(void *clientp, + curl_socket_t curlfd, + curlsocktype purpose); + +struct curl_sockaddr { + int family; + int socktype; + int protocol; + unsigned int addrlen; /* addrlen was a socklen_t type before 7.18.0 but it + turned really ugly and painful on the systems that + lack this type */ + struct sockaddr addr; +}; + +typedef curl_socket_t +(*curl_opensocket_callback)(void *clientp, + curlsocktype purpose, + struct curl_sockaddr *address); + +typedef int +(*curl_closesocket_callback)(void *clientp, curl_socket_t item); + +typedef enum { + CURLIOE_OK, /* I/O operation successful */ + CURLIOE_UNKNOWNCMD, /* command was unknown to callback */ + CURLIOE_FAILRESTART, /* failed to restart the read */ + CURLIOE_LAST /* never use */ +} curlioerr; + +typedef enum { + CURLIOCMD_NOP, /* no operation */ + CURLIOCMD_RESTARTREAD, /* restart the read stream from start */ + CURLIOCMD_LAST /* never use */ +} curliocmd; + +typedef curlioerr (*curl_ioctl_callback)(CURL *handle, + int cmd, + void *clientp); + +/* + * The following typedef's are signatures of malloc, free, realloc, strdup and + * calloc respectively. Function pointers of these types can be passed to the + * curl_global_init_mem() function to set user defined memory management + * callback routines. + */ +typedef void *(*curl_malloc_callback)(size_t size); +typedef void (*curl_free_callback)(void *ptr); +typedef void *(*curl_realloc_callback)(void *ptr, size_t size); +typedef char *(*curl_strdup_callback)(const char *str); +typedef void *(*curl_calloc_callback)(size_t nmemb, size_t size); + +/* the kind of data that is passed to information_callback*/ +typedef enum { + CURLINFO_TEXT = 0, + CURLINFO_HEADER_IN, /* 1 */ + CURLINFO_HEADER_OUT, /* 2 */ + CURLINFO_DATA_IN, /* 3 */ + CURLINFO_DATA_OUT, /* 4 */ + CURLINFO_SSL_DATA_IN, /* 5 */ + CURLINFO_SSL_DATA_OUT, /* 6 */ + CURLINFO_END +} curl_infotype; + +typedef int (*curl_debug_callback) + (CURL *handle, /* the handle/transfer this concerns */ + curl_infotype type, /* what kind of data */ + char *data, /* points to the data */ + size_t size, /* size of the data pointed to */ + void *userptr); /* whatever the user please */ + +/* All possible error codes from all sorts of curl functions. Future versions + may return other values, stay prepared. + + Always add new return codes last. Never *EVER* remove any. The return + codes must remain the same! + */ + +typedef enum { + CURLE_OK = 0, + CURLE_UNSUPPORTED_PROTOCOL, /* 1 */ + CURLE_FAILED_INIT, /* 2 */ + CURLE_URL_MALFORMAT, /* 3 */ + CURLE_NOT_BUILT_IN, /* 4 - [was obsoleted in August 2007 for + 7.17.0, reused in April 2011 for 7.21.5] */ + CURLE_COULDNT_RESOLVE_PROXY, /* 5 */ + CURLE_COULDNT_RESOLVE_HOST, /* 6 */ + CURLE_COULDNT_CONNECT, /* 7 */ + CURLE_FTP_WEIRD_SERVER_REPLY, /* 8 */ + CURLE_REMOTE_ACCESS_DENIED, /* 9 a service was denied by the server + due to lack of access - when login fails + this is not returned. */ + CURLE_FTP_ACCEPT_FAILED, /* 10 - [was obsoleted in April 2006 for + 7.15.4, reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASS_REPLY, /* 11 */ + CURLE_FTP_ACCEPT_TIMEOUT, /* 12 - timeout occurred accepting server + [was obsoleted in August 2007 for 7.17.0, + reused in Dec 2011 for 7.24.0]*/ + CURLE_FTP_WEIRD_PASV_REPLY, /* 13 */ + CURLE_FTP_WEIRD_227_FORMAT, /* 14 */ + CURLE_FTP_CANT_GET_HOST, /* 15 */ + CURLE_HTTP2, /* 16 - A problem in the http2 framing layer. + [was obsoleted in August 2007 for 7.17.0, + reused in July 2014 for 7.38.0] */ + CURLE_FTP_COULDNT_SET_TYPE, /* 17 */ + CURLE_PARTIAL_FILE, /* 18 */ + CURLE_FTP_COULDNT_RETR_FILE, /* 19 */ + CURLE_OBSOLETE20, /* 20 - NOT USED */ + CURLE_QUOTE_ERROR, /* 21 - quote command failure */ + CURLE_HTTP_RETURNED_ERROR, /* 22 */ + CURLE_WRITE_ERROR, /* 23 */ + CURLE_OBSOLETE24, /* 24 - NOT USED */ + CURLE_UPLOAD_FAILED, /* 25 - failed upload "command" */ + CURLE_READ_ERROR, /* 26 - couldn't open/read from file */ + CURLE_OUT_OF_MEMORY, /* 27 */ + /* Note: CURLE_OUT_OF_MEMORY may sometimes indicate a conversion error + instead of a memory allocation error if CURL_DOES_CONVERSIONS + is defined + */ + CURLE_OPERATION_TIMEDOUT, /* 28 - the timeout time was reached */ + CURLE_OBSOLETE29, /* 29 - NOT USED */ + CURLE_FTP_PORT_FAILED, /* 30 - FTP PORT operation failed */ + CURLE_FTP_COULDNT_USE_REST, /* 31 - the REST command failed */ + CURLE_OBSOLETE32, /* 32 - NOT USED */ + CURLE_RANGE_ERROR, /* 33 - RANGE "command" didn't work */ + CURLE_HTTP_POST_ERROR, /* 34 */ + CURLE_SSL_CONNECT_ERROR, /* 35 - wrong when connecting with SSL */ + CURLE_BAD_DOWNLOAD_RESUME, /* 36 - couldn't resume download */ + CURLE_FILE_COULDNT_READ_FILE, /* 37 */ + CURLE_LDAP_CANNOT_BIND, /* 38 */ + CURLE_LDAP_SEARCH_FAILED, /* 39 */ + CURLE_OBSOLETE40, /* 40 - NOT USED */ + CURLE_FUNCTION_NOT_FOUND, /* 41 */ + CURLE_ABORTED_BY_CALLBACK, /* 42 */ + CURLE_BAD_FUNCTION_ARGUMENT, /* 43 */ + CURLE_OBSOLETE44, /* 44 - NOT USED */ + CURLE_INTERFACE_FAILED, /* 45 - CURLOPT_INTERFACE failed */ + CURLE_OBSOLETE46, /* 46 - NOT USED */ + CURLE_TOO_MANY_REDIRECTS , /* 47 - catch endless re-direct loops */ + CURLE_UNKNOWN_OPTION, /* 48 - User specified an unknown option */ + CURLE_TELNET_OPTION_SYNTAX , /* 49 - Malformed telnet option */ + CURLE_OBSOLETE50, /* 50 - NOT USED */ + CURLE_PEER_FAILED_VERIFICATION, /* 51 - peer's certificate or fingerprint + wasn't verified fine */ + CURLE_GOT_NOTHING, /* 52 - when this is a specific error */ + CURLE_SSL_ENGINE_NOTFOUND, /* 53 - SSL crypto engine not found */ + CURLE_SSL_ENGINE_SETFAILED, /* 54 - can not set SSL crypto engine as + default */ + CURLE_SEND_ERROR, /* 55 - failed sending network data */ + CURLE_RECV_ERROR, /* 56 - failure in receiving network data */ + CURLE_OBSOLETE57, /* 57 - NOT IN USE */ + CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */ + CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */ + CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */ + CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized/bad encoding */ + CURLE_LDAP_INVALID_URL, /* 62 - Invalid LDAP URL */ + CURLE_FILESIZE_EXCEEDED, /* 63 - Maximum file size exceeded */ + CURLE_USE_SSL_FAILED, /* 64 - Requested FTP SSL level failed */ + CURLE_SEND_FAIL_REWIND, /* 65 - Sending the data requires a rewind + that failed */ + CURLE_SSL_ENGINE_INITFAILED, /* 66 - failed to initialise ENGINE */ + CURLE_LOGIN_DENIED, /* 67 - user, password or similar was not + accepted and we failed to login */ + CURLE_TFTP_NOTFOUND, /* 68 - file not found on server */ + CURLE_TFTP_PERM, /* 69 - permission problem on server */ + CURLE_REMOTE_DISK_FULL, /* 70 - out of disk space on server */ + CURLE_TFTP_ILLEGAL, /* 71 - Illegal TFTP operation */ + CURLE_TFTP_UNKNOWNID, /* 72 - Unknown transfer ID */ + CURLE_REMOTE_FILE_EXISTS, /* 73 - File already exists */ + CURLE_TFTP_NOSUCHUSER, /* 74 - No such user */ + CURLE_CONV_FAILED, /* 75 - conversion failed */ + CURLE_CONV_REQD, /* 76 - caller must register conversion + callbacks using curl_easy_setopt options + CURLOPT_CONV_FROM_NETWORK_FUNCTION, + CURLOPT_CONV_TO_NETWORK_FUNCTION, and + CURLOPT_CONV_FROM_UTF8_FUNCTION */ + CURLE_SSL_CACERT_BADFILE, /* 77 - could not load CACERT file, missing + or wrong format */ + CURLE_REMOTE_FILE_NOT_FOUND, /* 78 - remote file not found */ + CURLE_SSH, /* 79 - error from the SSH layer, somewhat + generic so the error message will be of + interest when this has happened */ + + CURLE_SSL_SHUTDOWN_FAILED, /* 80 - Failed to shut down the SSL + connection */ + CURLE_AGAIN, /* 81 - socket is not ready for send/recv, + wait till it's ready and try again (Added + in 7.18.2) */ + CURLE_SSL_CRL_BADFILE, /* 82 - could not load CRL file, missing or + wrong format (Added in 7.19.0) */ + CURLE_SSL_ISSUER_ERROR, /* 83 - Issuer check failed. (Added in + 7.19.0) */ + CURLE_FTP_PRET_FAILED, /* 84 - a PRET command failed */ + CURLE_RTSP_CSEQ_ERROR, /* 85 - mismatch of RTSP CSeq numbers */ + CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Ids */ + CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ + CURLE_CHUNK_FAILED, /* 88 - chunk callback reported error */ + CURLE_NO_CONNECTION_AVAILABLE, /* 89 - No connection available, the + session will be queued */ + CURLE_SSL_PINNEDPUBKEYNOTMATCH, /* 90 - specified pinned public key did not + match */ + CURLE_SSL_INVALIDCERTSTATUS, /* 91 - invalid certificate status */ + CURL_LAST /* never use! */ +} CURLcode; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Previously obsolete error code re-used in 7.38.0 */ +#define CURLE_OBSOLETE16 CURLE_HTTP2 + +/* Previously obsolete error codes re-used in 7.24.0 */ +#define CURLE_OBSOLETE10 CURLE_FTP_ACCEPT_FAILED +#define CURLE_OBSOLETE12 CURLE_FTP_ACCEPT_TIMEOUT + +/* compatibility with older names */ +#define CURLOPT_ENCODING CURLOPT_ACCEPT_ENCODING + +/* The following were added in 7.21.5, April 2011 */ +#define CURLE_UNKNOWN_TELNET_OPTION CURLE_UNKNOWN_OPTION + +/* The following were added in 7.17.1 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_SSL_PEER_CERTIFICATE CURLE_PEER_FAILED_VERIFICATION + +/* The following were added in 7.17.0 */ +/* These are scheduled to disappear by 2009 */ +#define CURLE_OBSOLETE CURLE_OBSOLETE50 /* no one should be using this! */ +#define CURLE_BAD_PASSWORD_ENTERED CURLE_OBSOLETE46 +#define CURLE_BAD_CALLING_ORDER CURLE_OBSOLETE44 +#define CURLE_FTP_USER_PASSWORD_INCORRECT CURLE_OBSOLETE10 +#define CURLE_FTP_CANT_RECONNECT CURLE_OBSOLETE16 +#define CURLE_FTP_COULDNT_GET_SIZE CURLE_OBSOLETE32 +#define CURLE_FTP_COULDNT_SET_ASCII CURLE_OBSOLETE29 +#define CURLE_FTP_WEIRD_USER_REPLY CURLE_OBSOLETE12 +#define CURLE_FTP_WRITE_ERROR CURLE_OBSOLETE20 +#define CURLE_LIBRARY_NOT_FOUND CURLE_OBSOLETE40 +#define CURLE_MALFORMAT_USER CURLE_OBSOLETE24 +#define CURLE_SHARE_IN_USE CURLE_OBSOLETE57 +#define CURLE_URL_MALFORMAT_USER CURLE_NOT_BUILT_IN + +#define CURLE_FTP_ACCESS_DENIED CURLE_REMOTE_ACCESS_DENIED +#define CURLE_FTP_COULDNT_SET_BINARY CURLE_FTP_COULDNT_SET_TYPE +#define CURLE_FTP_QUOTE_ERROR CURLE_QUOTE_ERROR +#define CURLE_TFTP_DISKFULL CURLE_REMOTE_DISK_FULL +#define CURLE_TFTP_EXISTS CURLE_REMOTE_FILE_EXISTS +#define CURLE_HTTP_RANGE_ERROR CURLE_RANGE_ERROR +#define CURLE_FTP_SSL_FAILED CURLE_USE_SSL_FAILED + +/* The following were added earlier */ + +#define CURLE_OPERATION_TIMEOUTED CURLE_OPERATION_TIMEDOUT + +#define CURLE_HTTP_NOT_FOUND CURLE_HTTP_RETURNED_ERROR +#define CURLE_HTTP_PORT_FAILED CURLE_INTERFACE_FAILED +#define CURLE_FTP_COULDNT_STOR_FILE CURLE_UPLOAD_FAILED + +#define CURLE_FTP_PARTIAL_FILE CURLE_PARTIAL_FILE +#define CURLE_FTP_BAD_DOWNLOAD_RESUME CURLE_BAD_DOWNLOAD_RESUME + +/* This was the error code 50 in 7.7.3 and a few earlier versions, this + is no longer used by libcurl but is instead #defined here only to not + make programs break */ +#define CURLE_ALREADY_COMPLETE 99999 + +/* Provide defines for really old option names */ +#define CURLOPT_FILE CURLOPT_WRITEDATA /* name changed in 7.9.7 */ +#define CURLOPT_INFILE CURLOPT_READDATA /* name changed in 7.9.7 */ +#define CURLOPT_WRITEHEADER CURLOPT_HEADERDATA + +/* Since long deprecated options with no code in the lib that does anything + with them. */ +#define CURLOPT_WRITEINFO CURLOPT_OBSOLETE40 +#define CURLOPT_CLOSEPOLICY CURLOPT_OBSOLETE72 + +#endif /*!CURL_NO_OLDIES*/ + +/* This prototype applies to all conversion callbacks */ +typedef CURLcode (*curl_conv_callback)(char *buffer, size_t length); + +typedef CURLcode (*curl_ssl_ctx_callback)(CURL *curl, /* easy handle */ + void *ssl_ctx, /* actually an + OpenSSL SSL_CTX */ + void *userptr); + +typedef enum { + CURLPROXY_HTTP = 0, /* added in 7.10, new in 7.19.4 default is to use + CONNECT HTTP/1.1 */ + CURLPROXY_HTTP_1_0 = 1, /* added in 7.19.4, force to use CONNECT + HTTP/1.0 */ + CURLPROXY_SOCKS4 = 4, /* support added in 7.15.2, enum existed already + in 7.10 */ + CURLPROXY_SOCKS5 = 5, /* added in 7.10 */ + CURLPROXY_SOCKS4A = 6, /* added in 7.18.0 */ + CURLPROXY_SOCKS5_HOSTNAME = 7 /* Use the SOCKS5 protocol but pass along the + host name rather than the IP address. added + in 7.18.0 */ +} curl_proxytype; /* this enum was added in 7.10 */ + +/* + * Bitmasks for CURLOPT_HTTPAUTH and CURLOPT_PROXYAUTH options: + * + * CURLAUTH_NONE - No HTTP authentication + * CURLAUTH_BASIC - HTTP Basic authentication (default) + * CURLAUTH_DIGEST - HTTP Digest authentication + * CURLAUTH_NEGOTIATE - HTTP Negotiate (SPNEGO) authentication + * CURLAUTH_GSSNEGOTIATE - Alias for CURLAUTH_NEGOTIATE (deprecated) + * CURLAUTH_NTLM - HTTP NTLM authentication + * CURLAUTH_DIGEST_IE - HTTP Digest authentication with IE flavour + * CURLAUTH_NTLM_WB - HTTP NTLM authentication delegated to winbind helper + * CURLAUTH_ONLY - Use together with a single other type to force no + * authentication or just that single type + * CURLAUTH_ANY - All fine types set + * CURLAUTH_ANYSAFE - All fine types except Basic + */ + +#define CURLAUTH_NONE ((unsigned long)0) +#define CURLAUTH_BASIC (((unsigned long)1)<<0) +#define CURLAUTH_DIGEST (((unsigned long)1)<<1) +#define CURLAUTH_NEGOTIATE (((unsigned long)1)<<2) +/* Deprecated since the advent of CURLAUTH_NEGOTIATE */ +#define CURLAUTH_GSSNEGOTIATE CURLAUTH_NEGOTIATE +#define CURLAUTH_NTLM (((unsigned long)1)<<3) +#define CURLAUTH_DIGEST_IE (((unsigned long)1)<<4) +#define CURLAUTH_NTLM_WB (((unsigned long)1)<<5) +#define CURLAUTH_ONLY (((unsigned long)1)<<31) +#define CURLAUTH_ANY (~CURLAUTH_DIGEST_IE) +#define CURLAUTH_ANYSAFE (~(CURLAUTH_BASIC|CURLAUTH_DIGEST_IE)) + +#define CURLSSH_AUTH_ANY ~0 /* all types supported by the server */ +#define CURLSSH_AUTH_NONE 0 /* none allowed, silly but complete */ +#define CURLSSH_AUTH_PUBLICKEY (1<<0) /* public/private key files */ +#define CURLSSH_AUTH_PASSWORD (1<<1) /* password */ +#define CURLSSH_AUTH_HOST (1<<2) /* host key files */ +#define CURLSSH_AUTH_KEYBOARD (1<<3) /* keyboard interactive */ +#define CURLSSH_AUTH_AGENT (1<<4) /* agent (ssh-agent, pageant...) */ +#define CURLSSH_AUTH_DEFAULT CURLSSH_AUTH_ANY + +#define CURLGSSAPI_DELEGATION_NONE 0 /* no delegation (default) */ +#define CURLGSSAPI_DELEGATION_POLICY_FLAG (1<<0) /* if permitted by policy */ +#define CURLGSSAPI_DELEGATION_FLAG (1<<1) /* delegate always */ + +#define CURL_ERROR_SIZE 256 + +enum curl_khtype { + CURLKHTYPE_UNKNOWN, + CURLKHTYPE_RSA1, + CURLKHTYPE_RSA, + CURLKHTYPE_DSS +}; + +struct curl_khkey { + const char *key; /* points to a zero-terminated string encoded with base64 + if len is zero, otherwise to the "raw" data */ + size_t len; + enum curl_khtype keytype; +}; + +/* this is the set of return values expected from the curl_sshkeycallback + callback */ +enum curl_khstat { + CURLKHSTAT_FINE_ADD_TO_FILE, + CURLKHSTAT_FINE, + CURLKHSTAT_REJECT, /* reject the connection, return an error */ + CURLKHSTAT_DEFER, /* do not accept it, but we can't answer right now so + this causes a CURLE_DEFER error but otherwise the + connection will be left intact etc */ + CURLKHSTAT_LAST /* not for use, only a marker for last-in-list */ +}; + +/* this is the set of status codes pass in to the callback */ +enum curl_khmatch { + CURLKHMATCH_OK, /* match */ + CURLKHMATCH_MISMATCH, /* host found, key mismatch! */ + CURLKHMATCH_MISSING, /* no matching host/key found */ + CURLKHMATCH_LAST /* not for use, only a marker for last-in-list */ +}; + +typedef int + (*curl_sshkeycallback) (CURL *easy, /* easy handle */ + const struct curl_khkey *knownkey, /* known */ + const struct curl_khkey *foundkey, /* found */ + enum curl_khmatch, /* libcurl's view on the keys */ + void *clientp); /* custom pointer passed from app */ + +/* parameter for the CURLOPT_USE_SSL option */ +typedef enum { + CURLUSESSL_NONE, /* do not attempt to use SSL */ + CURLUSESSL_TRY, /* try using SSL, proceed anyway otherwise */ + CURLUSESSL_CONTROL, /* SSL for the control connection or fail */ + CURLUSESSL_ALL, /* SSL for all communication or fail */ + CURLUSESSL_LAST /* not an option, never use */ +} curl_usessl; + +/* Definition of bits for the CURLOPT_SSL_OPTIONS argument: */ + +/* - ALLOW_BEAST tells libcurl to allow the BEAST SSL vulnerability in the + name of improving interoperability with older servers. Some SSL libraries + have introduced work-arounds for this flaw but those work-arounds sometimes + make the SSL communication fail. To regain functionality with those broken + servers, a user can this way allow the vulnerability back. */ +#define CURLSSLOPT_ALLOW_BEAST (1<<0) + +/* - NO_REVOKE tells libcurl to disable certificate revocation checks for those + SSL backends where such behavior is present. */ +#define CURLSSLOPT_NO_REVOKE (1<<1) + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2009 */ + +#define CURLFTPSSL_NONE CURLUSESSL_NONE +#define CURLFTPSSL_TRY CURLUSESSL_TRY +#define CURLFTPSSL_CONTROL CURLUSESSL_CONTROL +#define CURLFTPSSL_ALL CURLUSESSL_ALL +#define CURLFTPSSL_LAST CURLUSESSL_LAST +#define curl_ftpssl curl_usessl +#endif /*!CURL_NO_OLDIES*/ + +/* parameter for the CURLOPT_FTP_SSL_CCC option */ +typedef enum { + CURLFTPSSL_CCC_NONE, /* do not send CCC */ + CURLFTPSSL_CCC_PASSIVE, /* Let the server initiate the shutdown */ + CURLFTPSSL_CCC_ACTIVE, /* Initiate the shutdown */ + CURLFTPSSL_CCC_LAST /* not an option, never use */ +} curl_ftpccc; + +/* parameter for the CURLOPT_FTPSSLAUTH option */ +typedef enum { + CURLFTPAUTH_DEFAULT, /* let libcurl decide */ + CURLFTPAUTH_SSL, /* use "AUTH SSL" */ + CURLFTPAUTH_TLS, /* use "AUTH TLS" */ + CURLFTPAUTH_LAST /* not an option, never use */ +} curl_ftpauth; + +/* parameter for the CURLOPT_FTP_CREATE_MISSING_DIRS option */ +typedef enum { + CURLFTP_CREATE_DIR_NONE, /* do NOT create missing dirs! */ + CURLFTP_CREATE_DIR, /* (FTP/SFTP) if CWD fails, try MKD and then CWD + again if MKD succeeded, for SFTP this does + similar magic */ + CURLFTP_CREATE_DIR_RETRY, /* (FTP only) if CWD fails, try MKD and then CWD + again even if MKD failed! */ + CURLFTP_CREATE_DIR_LAST /* not an option, never use */ +} curl_ftpcreatedir; + +/* parameter for the CURLOPT_FTP_FILEMETHOD option */ +typedef enum { + CURLFTPMETHOD_DEFAULT, /* let libcurl pick */ + CURLFTPMETHOD_MULTICWD, /* single CWD operation for each path part */ + CURLFTPMETHOD_NOCWD, /* no CWD at all */ + CURLFTPMETHOD_SINGLECWD, /* one CWD to full dir, then work on file */ + CURLFTPMETHOD_LAST /* not an option, never use */ +} curl_ftpmethod; + +/* bitmask defines for CURLOPT_HEADEROPT */ +#define CURLHEADER_UNIFIED 0 +#define CURLHEADER_SEPARATE (1<<0) + +/* CURLPROTO_ defines are for the CURLOPT_*PROTOCOLS options */ +#define CURLPROTO_HTTP (1<<0) +#define CURLPROTO_HTTPS (1<<1) +#define CURLPROTO_FTP (1<<2) +#define CURLPROTO_FTPS (1<<3) +#define CURLPROTO_SCP (1<<4) +#define CURLPROTO_SFTP (1<<5) +#define CURLPROTO_TELNET (1<<6) +#define CURLPROTO_LDAP (1<<7) +#define CURLPROTO_LDAPS (1<<8) +#define CURLPROTO_DICT (1<<9) +#define CURLPROTO_FILE (1<<10) +#define CURLPROTO_TFTP (1<<11) +#define CURLPROTO_IMAP (1<<12) +#define CURLPROTO_IMAPS (1<<13) +#define CURLPROTO_POP3 (1<<14) +#define CURLPROTO_POP3S (1<<15) +#define CURLPROTO_SMTP (1<<16) +#define CURLPROTO_SMTPS (1<<17) +#define CURLPROTO_RTSP (1<<18) +#define CURLPROTO_RTMP (1<<19) +#define CURLPROTO_RTMPT (1<<20) +#define CURLPROTO_RTMPE (1<<21) +#define CURLPROTO_RTMPTE (1<<22) +#define CURLPROTO_RTMPS (1<<23) +#define CURLPROTO_RTMPTS (1<<24) +#define CURLPROTO_GOPHER (1<<25) +#define CURLPROTO_SMB (1<<26) +#define CURLPROTO_SMBS (1<<27) +#define CURLPROTO_ALL (~0) /* enable everything */ + +/* long may be 32 or 64 bits, but we should never depend on anything else + but 32 */ +#define CURLOPTTYPE_LONG 0 +#define CURLOPTTYPE_OBJECTPOINT 10000 +#define CURLOPTTYPE_STRINGPOINT 10000 +#define CURLOPTTYPE_FUNCTIONPOINT 20000 +#define CURLOPTTYPE_OFF_T 30000 + +/* *STRINGPOINT is an alias for OBJECTPOINT to allow tools to extract the + string options from the header file */ + +/* name is uppercase CURLOPT_, + type is one of the defined CURLOPTTYPE_ + number is unique identifier */ +#ifdef CINIT +#undef CINIT +#endif + +#ifdef CURL_ISOCPP +#define CINIT(na,t,nu) CURLOPT_ ## na = CURLOPTTYPE_ ## t + nu +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define STRINGPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLOPT_/**/name = type + number +#endif + +/* + * This macro-mania below setups the CURLOPT_[what] enum, to be used with + * curl_easy_setopt(). The first argument in the CINIT() macro is the [what] + * word. + */ + +typedef enum { + /* This is the FILE * or void * the regular output should be written to. */ + CINIT(WRITEDATA, OBJECTPOINT, 1), + + /* The full URL to get/put */ + CINIT(URL, STRINGPOINT, 2), + + /* Port number to connect to, if other than default. */ + CINIT(PORT, LONG, 3), + + /* Name of proxy to use. */ + CINIT(PROXY, STRINGPOINT, 4), + + /* "user:password;options" to use when fetching. */ + CINIT(USERPWD, STRINGPOINT, 5), + + /* "user:password" to use with proxy. */ + CINIT(PROXYUSERPWD, STRINGPOINT, 6), + + /* Range to get, specified as an ASCII string. */ + CINIT(RANGE, STRINGPOINT, 7), + + /* not used */ + + /* Specified file stream to upload from (use as input): */ + CINIT(READDATA, OBJECTPOINT, 9), + + /* Buffer to receive error messages in, must be at least CURL_ERROR_SIZE + * bytes big. If this is not used, error messages go to stderr instead: */ + CINIT(ERRORBUFFER, OBJECTPOINT, 10), + + /* Function that will be called to store the output (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11), + + /* Function that will be called to read the input (instead of fread). The + * parameters will use fread() syntax, make sure to follow them. */ + CINIT(READFUNCTION, FUNCTIONPOINT, 12), + + /* Time-out the read operation after this amount of seconds */ + CINIT(TIMEOUT, LONG, 13), + + /* If the CURLOPT_INFILE is used, this can be used to inform libcurl about + * how large the file being sent really is. That allows better error + * checking and better verifies that the upload was successful. -1 means + * unknown size. + * + * For large file support, there is also a _LARGE version of the key + * which takes an off_t type, allowing platforms with larger off_t + * sizes to handle larger files. See below for INFILESIZE_LARGE. + */ + CINIT(INFILESIZE, LONG, 14), + + /* POST static input fields. */ + CINIT(POSTFIELDS, OBJECTPOINT, 15), + + /* Set the referrer page (needed by some CGIs) */ + CINIT(REFERER, STRINGPOINT, 16), + + /* Set the FTP PORT string (interface name, named or numerical IP address) + Use i.e '-' to use default address. */ + CINIT(FTPPORT, STRINGPOINT, 17), + + /* Set the User-Agent string (examined by some CGIs) */ + CINIT(USERAGENT, STRINGPOINT, 18), + + /* If the download receives less than "low speed limit" bytes/second + * during "low speed time" seconds, the operations is aborted. + * You could i.e if you have a pretty high speed connection, abort if + * it is less than 2000 bytes/sec during 20 seconds. + */ + + /* Set the "low speed limit" */ + CINIT(LOW_SPEED_LIMIT, LONG, 19), + + /* Set the "low speed time" */ + CINIT(LOW_SPEED_TIME, LONG, 20), + + /* Set the continuation offset. + * + * Note there is also a _LARGE version of this key which uses + * off_t types, allowing for large file offsets on platforms which + * use larger-than-32-bit off_t's. Look below for RESUME_FROM_LARGE. + */ + CINIT(RESUME_FROM, LONG, 21), + + /* Set cookie in request: */ + CINIT(COOKIE, STRINGPOINT, 22), + + /* This points to a linked list of headers, struct curl_slist kind. This + list is also used for RTSP (in spite of its name) */ + CINIT(HTTPHEADER, OBJECTPOINT, 23), + + /* This points to a linked list of post entries, struct curl_httppost */ + CINIT(HTTPPOST, OBJECTPOINT, 24), + + /* name of the file keeping your private SSL-certificate */ + CINIT(SSLCERT, STRINGPOINT, 25), + + /* password for the SSL or SSH private key */ + CINIT(KEYPASSWD, STRINGPOINT, 26), + + /* send TYPE parameter? */ + CINIT(CRLF, LONG, 27), + + /* send linked-list of QUOTE commands */ + CINIT(QUOTE, OBJECTPOINT, 28), + + /* send FILE * or void * to store headers to, if you use a callback it + is simply passed to the callback unmodified */ + CINIT(HEADERDATA, OBJECTPOINT, 29), + + /* point to a file to read the initial cookies from, also enables + "cookie awareness" */ + CINIT(COOKIEFILE, STRINGPOINT, 31), + + /* What version to specifically try to use. + See CURL_SSLVERSION defines below. */ + CINIT(SSLVERSION, LONG, 32), + + /* What kind of HTTP time condition to use, see defines */ + CINIT(TIMECONDITION, LONG, 33), + + /* Time to use with the above condition. Specified in number of seconds + since 1 Jan 1970 */ + CINIT(TIMEVALUE, LONG, 34), + + /* 35 = OBSOLETE */ + + /* Custom request, for customizing the get command like + HTTP: DELETE, TRACE and others + FTP: to use a different list command + */ + CINIT(CUSTOMREQUEST, STRINGPOINT, 36), + + /* FILE handle to use instead of stderr */ + CINIT(STDERR, OBJECTPOINT, 37), + + /* 38 is not used */ + + /* send linked-list of post-transfer QUOTE commands */ + CINIT(POSTQUOTE, OBJECTPOINT, 39), + + CINIT(OBSOLETE40, OBJECTPOINT, 40), /* OBSOLETE, do not use! */ + + CINIT(VERBOSE, LONG, 41), /* talk a lot */ + CINIT(HEADER, LONG, 42), /* throw the header out too */ + CINIT(NOPROGRESS, LONG, 43), /* shut off the progress meter */ + CINIT(NOBODY, LONG, 44), /* use HEAD to get http document */ + CINIT(FAILONERROR, LONG, 45), /* no output on http error codes >= 400 */ + CINIT(UPLOAD, LONG, 46), /* this is an upload */ + CINIT(POST, LONG, 47), /* HTTP POST method */ + CINIT(DIRLISTONLY, LONG, 48), /* bare names when listing directories */ + + CINIT(APPEND, LONG, 50), /* Append instead of overwrite on upload! */ + + /* Specify whether to read the user+password from the .netrc or the URL. + * This must be one of the CURL_NETRC_* enums below. */ + CINIT(NETRC, LONG, 51), + + CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */ + + CINIT(TRANSFERTEXT, LONG, 53), /* transfer data in text/ASCII format */ + CINIT(PUT, LONG, 54), /* HTTP PUT */ + + /* 55 = OBSOLETE */ + + /* DEPRECATED + * Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_progress_callback + * prototype defines. */ + CINIT(PROGRESSFUNCTION, FUNCTIONPOINT, 56), + + /* Data passed to the CURLOPT_PROGRESSFUNCTION and CURLOPT_XFERINFOFUNCTION + callbacks */ + CINIT(PROGRESSDATA, OBJECTPOINT, 57), +#define CURLOPT_XFERINFODATA CURLOPT_PROGRESSDATA + + /* We want the referrer field set automatically when following locations */ + CINIT(AUTOREFERER, LONG, 58), + + /* Port of the proxy, can be set in the proxy string as well with: + "[host]:[port]" */ + CINIT(PROXYPORT, LONG, 59), + + /* size of the POST input data, if strlen() is not good to use */ + CINIT(POSTFIELDSIZE, LONG, 60), + + /* tunnel non-http operations through a HTTP proxy */ + CINIT(HTTPPROXYTUNNEL, LONG, 61), + + /* Set the interface string to use as outgoing network interface */ + CINIT(INTERFACE, STRINGPOINT, 62), + + /* Set the krb4/5 security level, this also enables krb4/5 awareness. This + * is a string, 'clear', 'safe', 'confidential' or 'private'. If the string + * is set but doesn't match one of these, 'private' will be used. */ + CINIT(KRBLEVEL, STRINGPOINT, 63), + + /* Set if we should verify the peer in ssl handshake, set 1 to verify. */ + CINIT(SSL_VERIFYPEER, LONG, 64), + + /* The CApath or CAfile used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAINFO, STRINGPOINT, 65), + + /* 66 = OBSOLETE */ + /* 67 = OBSOLETE */ + + /* Maximum number of http redirects to follow */ + CINIT(MAXREDIRS, LONG, 68), + + /* Pass a long set to 1 to get the date of the requested document (if + possible)! Pass a zero to shut it off. */ + CINIT(FILETIME, LONG, 69), + + /* This points to a linked list of telnet options */ + CINIT(TELNETOPTIONS, OBJECTPOINT, 70), + + /* Max amount of cached alive connections */ + CINIT(MAXCONNECTS, LONG, 71), + + CINIT(OBSOLETE72, LONG, 72), /* OBSOLETE, do not use! */ + + /* 73 = OBSOLETE */ + + /* Set to explicitly use a new connection for the upcoming transfer. + Do not use this unless you're absolutely sure of this, as it makes the + operation slower and is less friendly for the network. */ + CINIT(FRESH_CONNECT, LONG, 74), + + /* Set to explicitly forbid the upcoming transfer's connection to be re-used + when done. Do not use this unless you're absolutely sure of this, as it + makes the operation slower and is less friendly for the network. */ + CINIT(FORBID_REUSE, LONG, 75), + + /* Set to a file name that contains random data for libcurl to use to + seed the random engine when doing SSL connects. */ + CINIT(RANDOM_FILE, STRINGPOINT, 76), + + /* Set to the Entropy Gathering Daemon socket pathname */ + CINIT(EGDSOCKET, STRINGPOINT, 77), + + /* Time-out connect operations after this amount of seconds, if connects are + OK within this time, then fine... This only aborts the connect phase. */ + CINIT(CONNECTTIMEOUT, LONG, 78), + + /* Function that will be called to store headers (instead of fwrite). The + * parameters will use fwrite() syntax, make sure to follow them. */ + CINIT(HEADERFUNCTION, FUNCTIONPOINT, 79), + + /* Set this to force the HTTP request to get back to GET. Only really usable + if POST, PUT or a custom request have been used first. + */ + CINIT(HTTPGET, LONG, 80), + + /* Set if we should verify the Common name from the peer certificate in ssl + * handshake, set 1 to check existence, 2 to ensure that it matches the + * provided hostname. */ + CINIT(SSL_VERIFYHOST, LONG, 81), + + /* Specify which file name to write all known cookies in after completed + operation. Set file name to "-" (dash) to make it go to stdout. */ + CINIT(COOKIEJAR, STRINGPOINT, 82), + + /* Specify which SSL ciphers to use */ + CINIT(SSL_CIPHER_LIST, STRINGPOINT, 83), + + /* Specify which HTTP version to use! This must be set to one of the + CURL_HTTP_VERSION* enums set below. */ + CINIT(HTTP_VERSION, LONG, 84), + + /* Specifically switch on or off the FTP engine's use of the EPSV command. By + default, that one will always be attempted before the more traditional + PASV command. */ + CINIT(FTP_USE_EPSV, LONG, 85), + + /* type of the file keeping your SSL-certificate ("DER", "PEM", "ENG") */ + CINIT(SSLCERTTYPE, STRINGPOINT, 86), + + /* name of the file keeping your private SSL-key */ + CINIT(SSLKEY, STRINGPOINT, 87), + + /* type of the file keeping your private SSL-key ("DER", "PEM", "ENG") */ + CINIT(SSLKEYTYPE, STRINGPOINT, 88), + + /* crypto engine for the SSL-sub system */ + CINIT(SSLENGINE, STRINGPOINT, 89), + + /* set the crypto engine for the SSL-sub system as default + the param has no meaning... + */ + CINIT(SSLENGINE_DEFAULT, LONG, 90), + + /* Non-zero value means to use the global dns cache */ + CINIT(DNS_USE_GLOBAL_CACHE, LONG, 91), /* DEPRECATED, do not use! */ + + /* DNS cache timeout */ + CINIT(DNS_CACHE_TIMEOUT, LONG, 92), + + /* send linked-list of pre-transfer QUOTE commands */ + CINIT(PREQUOTE, OBJECTPOINT, 93), + + /* set the debug function */ + CINIT(DEBUGFUNCTION, FUNCTIONPOINT, 94), + + /* set the data for the debug function */ + CINIT(DEBUGDATA, OBJECTPOINT, 95), + + /* mark this as start of a cookie session */ + CINIT(COOKIESESSION, LONG, 96), + + /* The CApath directory used to validate the peer certificate + this option is used only if SSL_VERIFYPEER is true */ + CINIT(CAPATH, STRINGPOINT, 97), + + /* Instruct libcurl to use a smaller receive buffer */ + CINIT(BUFFERSIZE, LONG, 98), + + /* Instruct libcurl to not use any signal/alarm handlers, even when using + timeouts. This option is useful for multi-threaded applications. + See libcurl-the-guide for more background information. */ + CINIT(NOSIGNAL, LONG, 99), + + /* Provide a CURLShare for mutexing non-ts data */ + CINIT(SHARE, OBJECTPOINT, 100), + + /* indicates type of proxy. accepted values are CURLPROXY_HTTP (default), + CURLPROXY_SOCKS4, CURLPROXY_SOCKS4A and CURLPROXY_SOCKS5. */ + CINIT(PROXYTYPE, LONG, 101), + + /* Set the Accept-Encoding string. Use this to tell a server you would like + the response to be compressed. Before 7.21.6, this was known as + CURLOPT_ENCODING */ + CINIT(ACCEPT_ENCODING, STRINGPOINT, 102), + + /* Set pointer to private data */ + CINIT(PRIVATE, OBJECTPOINT, 103), + + /* Set aliases for HTTP 200 in the HTTP Response header */ + CINIT(HTTP200ALIASES, OBJECTPOINT, 104), + + /* Continue to send authentication (user+password) when following locations, + even when hostname changed. This can potentially send off the name + and password to whatever host the server decides. */ + CINIT(UNRESTRICTED_AUTH, LONG, 105), + + /* Specifically switch on or off the FTP engine's use of the EPRT command ( + it also disables the LPRT attempt). By default, those ones will always be + attempted before the good old traditional PORT command. */ + CINIT(FTP_USE_EPRT, LONG, 106), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_USERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(HTTPAUTH, LONG, 107), + + /* Set the ssl context callback function, currently only for OpenSSL ssl_ctx + in second argument. The function must be matching the + curl_ssl_ctx_callback proto. */ + CINIT(SSL_CTX_FUNCTION, FUNCTIONPOINT, 108), + + /* Set the userdata for the ssl context callback function's third + argument */ + CINIT(SSL_CTX_DATA, OBJECTPOINT, 109), + + /* FTP Option that causes missing dirs to be created on the remote server. + In 7.19.4 we introduced the convenience enums for this option using the + CURLFTP_CREATE_DIR prefix. + */ + CINIT(FTP_CREATE_MISSING_DIRS, LONG, 110), + + /* Set this to a bitmask value to enable the particular authentications + methods you like. Use this in combination with CURLOPT_PROXYUSERPWD. + Note that setting multiple bits may cause extra network round-trips. */ + CINIT(PROXYAUTH, LONG, 111), + + /* FTP option that changes the timeout, in seconds, associated with + getting a response. This is different from transfer timeout time and + essentially places a demand on the FTP server to acknowledge commands + in a timely manner. */ + CINIT(FTP_RESPONSE_TIMEOUT, LONG, 112), +#define CURLOPT_SERVER_RESPONSE_TIMEOUT CURLOPT_FTP_RESPONSE_TIMEOUT + + /* Set this option to one of the CURL_IPRESOLVE_* defines (see below) to + tell libcurl to resolve names to those IP versions only. This only has + affect on systems with support for more than one, i.e IPv4 _and_ IPv6. */ + CINIT(IPRESOLVE, LONG, 113), + + /* Set this option to limit the size of a file that will be downloaded from + an HTTP or FTP server. + + Note there is also _LARGE version which adds large file support for + platforms which have larger off_t sizes. See MAXFILESIZE_LARGE below. */ + CINIT(MAXFILESIZE, LONG, 114), + + /* See the comment for INFILESIZE above, but in short, specifies + * the size of the file being uploaded. -1 means unknown. + */ + CINIT(INFILESIZE_LARGE, OFF_T, 115), + + /* Sets the continuation offset. There is also a LONG version of this; + * look above for RESUME_FROM. + */ + CINIT(RESUME_FROM_LARGE, OFF_T, 116), + + /* Sets the maximum size of data that will be downloaded from + * an HTTP or FTP server. See MAXFILESIZE above for the LONG version. + */ + CINIT(MAXFILESIZE_LARGE, OFF_T, 117), + + /* Set this option to the file name of your .netrc file you want libcurl + to parse (using the CURLOPT_NETRC option). If not set, libcurl will do + a poor attempt to find the user's home directory and check for a .netrc + file in there. */ + CINIT(NETRC_FILE, STRINGPOINT, 118), + + /* Enable SSL/TLS for FTP, pick one of: + CURLUSESSL_TRY - try using SSL, proceed anyway otherwise + CURLUSESSL_CONTROL - SSL for the control connection or fail + CURLUSESSL_ALL - SSL for all communication or fail + */ + CINIT(USE_SSL, LONG, 119), + + /* The _LARGE version of the standard POSTFIELDSIZE option */ + CINIT(POSTFIELDSIZE_LARGE, OFF_T, 120), + + /* Enable/disable the TCP Nagle algorithm */ + CINIT(TCP_NODELAY, LONG, 121), + + /* 122 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 123 OBSOLETE. Gone in 7.16.0 */ + /* 124 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 125 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 126 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ + /* 127 OBSOLETE. Gone in 7.16.0 */ + /* 128 OBSOLETE. Gone in 7.16.0 */ + + /* When FTP over SSL/TLS is selected (with CURLOPT_USE_SSL), this option + can be used to change libcurl's default action which is to first try + "AUTH SSL" and then "AUTH TLS" in this order, and proceed when a OK + response has been received. + + Available parameters are: + CURLFTPAUTH_DEFAULT - let libcurl decide + CURLFTPAUTH_SSL - try "AUTH SSL" first, then TLS + CURLFTPAUTH_TLS - try "AUTH TLS" first, then SSL + */ + CINIT(FTPSSLAUTH, LONG, 129), + + CINIT(IOCTLFUNCTION, FUNCTIONPOINT, 130), + CINIT(IOCTLDATA, OBJECTPOINT, 131), + + /* 132 OBSOLETE. Gone in 7.16.0 */ + /* 133 OBSOLETE. Gone in 7.16.0 */ + + /* zero terminated string for pass on to the FTP server when asked for + "account" info */ + CINIT(FTP_ACCOUNT, STRINGPOINT, 134), + + /* feed cookie into cookie engine */ + CINIT(COOKIELIST, STRINGPOINT, 135), + + /* ignore Content-Length */ + CINIT(IGNORE_CONTENT_LENGTH, LONG, 136), + + /* Set to non-zero to skip the IP address received in a 227 PASV FTP server + response. Typically used for FTP-SSL purposes but is not restricted to + that. libcurl will then instead use the same IP address it used for the + control connection. */ + CINIT(FTP_SKIP_PASV_IP, LONG, 137), + + /* Select "file method" to use when doing FTP, see the curl_ftpmethod + above. */ + CINIT(FTP_FILEMETHOD, LONG, 138), + + /* Local port number to bind the socket to */ + CINIT(LOCALPORT, LONG, 139), + + /* Number of ports to try, including the first one set with LOCALPORT. + Thus, setting it to 1 will make no additional attempts but the first. + */ + CINIT(LOCALPORTRANGE, LONG, 140), + + /* no transfer, set up connection and let application use the socket by + extracting it with CURLINFO_LASTSOCKET */ + CINIT(CONNECT_ONLY, LONG, 141), + + /* Function that will be called to convert from the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_FROM_NETWORK_FUNCTION, FUNCTIONPOINT, 142), + + /* Function that will be called to convert to the + network encoding (instead of using the iconv calls in libcurl) */ + CINIT(CONV_TO_NETWORK_FUNCTION, FUNCTIONPOINT, 143), + + /* Function that will be called to convert from UTF8 + (instead of using the iconv calls in libcurl) + Note that this is used only for SSL certificate processing */ + CINIT(CONV_FROM_UTF8_FUNCTION, FUNCTIONPOINT, 144), + + /* if the connection proceeds too quickly then need to slow it down */ + /* limit-rate: maximum number of bytes per second to send or receive */ + CINIT(MAX_SEND_SPEED_LARGE, OFF_T, 145), + CINIT(MAX_RECV_SPEED_LARGE, OFF_T, 146), + + /* Pointer to command string to send if USER/PASS fails. */ + CINIT(FTP_ALTERNATIVE_TO_USER, STRINGPOINT, 147), + + /* callback function for setting socket options */ + CINIT(SOCKOPTFUNCTION, FUNCTIONPOINT, 148), + CINIT(SOCKOPTDATA, OBJECTPOINT, 149), + + /* set to 0 to disable session ID re-use for this transfer, default is + enabled (== 1) */ + CINIT(SSL_SESSIONID_CACHE, LONG, 150), + + /* allowed SSH authentication methods */ + CINIT(SSH_AUTH_TYPES, LONG, 151), + + /* Used by scp/sftp to do public/private key authentication */ + CINIT(SSH_PUBLIC_KEYFILE, STRINGPOINT, 152), + CINIT(SSH_PRIVATE_KEYFILE, STRINGPOINT, 153), + + /* Send CCC (Clear Command Channel) after authentication */ + CINIT(FTP_SSL_CCC, LONG, 154), + + /* Same as TIMEOUT and CONNECTTIMEOUT, but with ms resolution */ + CINIT(TIMEOUT_MS, LONG, 155), + CINIT(CONNECTTIMEOUT_MS, LONG, 156), + + /* set to zero to disable the libcurl's decoding and thus pass the raw body + data to the application even when it is encoded/compressed */ + CINIT(HTTP_TRANSFER_DECODING, LONG, 157), + CINIT(HTTP_CONTENT_DECODING, LONG, 158), + + /* Permission used when creating new files and directories on the remote + server for protocols that support it, SFTP/SCP/FILE */ + CINIT(NEW_FILE_PERMS, LONG, 159), + CINIT(NEW_DIRECTORY_PERMS, LONG, 160), + + /* Set the behaviour of POST when redirecting. Values must be set to one + of CURL_REDIR* defines below. This used to be called CURLOPT_POST301 */ + CINIT(POSTREDIR, LONG, 161), + + /* used by scp/sftp to verify the host's public key */ + CINIT(SSH_HOST_PUBLIC_KEY_MD5, STRINGPOINT, 162), + + /* Callback function for opening socket (instead of socket(2)). Optionally, + callback is able change the address or refuse to connect returning + CURL_SOCKET_BAD. The callback should have type + curl_opensocket_callback */ + CINIT(OPENSOCKETFUNCTION, FUNCTIONPOINT, 163), + CINIT(OPENSOCKETDATA, OBJECTPOINT, 164), + + /* POST volatile input fields. */ + CINIT(COPYPOSTFIELDS, OBJECTPOINT, 165), + + /* set transfer mode (;type=) when doing FTP via an HTTP proxy */ + CINIT(PROXY_TRANSFER_MODE, LONG, 166), + + /* Callback function for seeking in the input stream */ + CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167), + CINIT(SEEKDATA, OBJECTPOINT, 168), + + /* CRL file */ + CINIT(CRLFILE, STRINGPOINT, 169), + + /* Issuer certificate */ + CINIT(ISSUERCERT, STRINGPOINT, 170), + + /* (IPv6) Address scope */ + CINIT(ADDRESS_SCOPE, LONG, 171), + + /* Collect certificate chain info and allow it to get retrievable with + CURLINFO_CERTINFO after the transfer is complete. */ + CINIT(CERTINFO, LONG, 172), + + /* "name" and "pwd" to use when fetching. */ + CINIT(USERNAME, STRINGPOINT, 173), + CINIT(PASSWORD, STRINGPOINT, 174), + + /* "name" and "pwd" to use with Proxy when fetching. */ + CINIT(PROXYUSERNAME, STRINGPOINT, 175), + CINIT(PROXYPASSWORD, STRINGPOINT, 176), + + /* Comma separated list of hostnames defining no-proxy zones. These should + match both hostnames directly, and hostnames within a domain. For + example, local.com will match local.com and www.local.com, but NOT + notlocal.com or www.notlocal.com. For compatibility with other + implementations of this, .local.com will be considered to be the same as + local.com. A single * is the only valid wildcard, and effectively + disables the use of proxy. */ + CINIT(NOPROXY, STRINGPOINT, 177), + + /* block size for TFTP transfers */ + CINIT(TFTP_BLKSIZE, LONG, 178), + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_SERVICE, STRINGPOINT, 179), + + /* Socks Service */ + CINIT(SOCKS5_GSSAPI_NEC, LONG, 180), + + /* set the bitmask for the protocols that are allowed to be used for the + transfer, which thus helps the app which takes URLs from users or other + external inputs and want to restrict what protocol(s) to deal + with. Defaults to CURLPROTO_ALL. */ + CINIT(PROTOCOLS, LONG, 181), + + /* set the bitmask for the protocols that libcurl is allowed to follow to, + as a subset of the CURLOPT_PROTOCOLS ones. That means the protocol needs + to be set in both bitmasks to be allowed to get redirected to. Defaults + to all protocols except FILE and SCP. */ + CINIT(REDIR_PROTOCOLS, LONG, 182), + + /* set the SSH knownhost file name to use */ + CINIT(SSH_KNOWNHOSTS, STRINGPOINT, 183), + + /* set the SSH host key callback, must point to a curl_sshkeycallback + function */ + CINIT(SSH_KEYFUNCTION, FUNCTIONPOINT, 184), + + /* set the SSH host key callback custom pointer */ + CINIT(SSH_KEYDATA, OBJECTPOINT, 185), + + /* set the SMTP mail originator */ + CINIT(MAIL_FROM, STRINGPOINT, 186), + + /* set the list of SMTP mail receiver(s) */ + CINIT(MAIL_RCPT, OBJECTPOINT, 187), + + /* FTP: send PRET before PASV */ + CINIT(FTP_USE_PRET, LONG, 188), + + /* RTSP request method (OPTIONS, SETUP, PLAY, etc...) */ + CINIT(RTSP_REQUEST, LONG, 189), + + /* The RTSP session identifier */ + CINIT(RTSP_SESSION_ID, STRINGPOINT, 190), + + /* The RTSP stream URI */ + CINIT(RTSP_STREAM_URI, STRINGPOINT, 191), + + /* The Transport: header to use in RTSP requests */ + CINIT(RTSP_TRANSPORT, STRINGPOINT, 192), + + /* Manually initialize the client RTSP CSeq for this handle */ + CINIT(RTSP_CLIENT_CSEQ, LONG, 193), + + /* Manually initialize the server RTSP CSeq for this handle */ + CINIT(RTSP_SERVER_CSEQ, LONG, 194), + + /* The stream to pass to INTERLEAVEFUNCTION. */ + CINIT(INTERLEAVEDATA, OBJECTPOINT, 195), + + /* Let the application define a custom write method for RTP data */ + CINIT(INTERLEAVEFUNCTION, FUNCTIONPOINT, 196), + + /* Turn on wildcard matching */ + CINIT(WILDCARDMATCH, LONG, 197), + + /* Directory matching callback called before downloading of an + individual file (chunk) started */ + CINIT(CHUNK_BGN_FUNCTION, FUNCTIONPOINT, 198), + + /* Directory matching callback called after the file (chunk) + was downloaded, or skipped */ + CINIT(CHUNK_END_FUNCTION, FUNCTIONPOINT, 199), + + /* Change match (fnmatch-like) callback for wildcard matching */ + CINIT(FNMATCH_FUNCTION, FUNCTIONPOINT, 200), + + /* Let the application define custom chunk data pointer */ + CINIT(CHUNK_DATA, OBJECTPOINT, 201), + + /* FNMATCH_FUNCTION user pointer */ + CINIT(FNMATCH_DATA, OBJECTPOINT, 202), + + /* send linked-list of name:port:address sets */ + CINIT(RESOLVE, OBJECTPOINT, 203), + + /* Set a username for authenticated TLS */ + CINIT(TLSAUTH_USERNAME, STRINGPOINT, 204), + + /* Set a password for authenticated TLS */ + CINIT(TLSAUTH_PASSWORD, STRINGPOINT, 205), + + /* Set authentication type for authenticated TLS */ + CINIT(TLSAUTH_TYPE, STRINGPOINT, 206), + + /* Set to 1 to enable the "TE:" header in HTTP requests to ask for + compressed transfer-encoded responses. Set to 0 to disable the use of TE: + in outgoing requests. The current default is 0, but it might change in a + future libcurl release. + + libcurl will ask for the compressed methods it knows of, and if that + isn't any, it will not ask for transfer-encoding at all even if this + option is set to 1. + + */ + CINIT(TRANSFER_ENCODING, LONG, 207), + + /* Callback function for closing socket (instead of close(2)). The callback + should have type curl_closesocket_callback */ + CINIT(CLOSESOCKETFUNCTION, FUNCTIONPOINT, 208), + CINIT(CLOSESOCKETDATA, OBJECTPOINT, 209), + + /* allow GSSAPI credential delegation */ + CINIT(GSSAPI_DELEGATION, LONG, 210), + + /* Set the name servers to use for DNS resolution */ + CINIT(DNS_SERVERS, STRINGPOINT, 211), + + /* Time-out accept operations (currently for FTP only) after this amount + of miliseconds. */ + CINIT(ACCEPTTIMEOUT_MS, LONG, 212), + + /* Set TCP keepalive */ + CINIT(TCP_KEEPALIVE, LONG, 213), + + /* non-universal keepalive knobs (Linux, AIX, HP-UX, more) */ + CINIT(TCP_KEEPIDLE, LONG, 214), + CINIT(TCP_KEEPINTVL, LONG, 215), + + /* Enable/disable specific SSL features with a bitmask, see CURLSSLOPT_* */ + CINIT(SSL_OPTIONS, LONG, 216), + + /* Set the SMTP auth originator */ + CINIT(MAIL_AUTH, STRINGPOINT, 217), + + /* Enable/disable SASL initial response */ + CINIT(SASL_IR, LONG, 218), + + /* Function that will be called instead of the internal progress display + * function. This function should be defined as the curl_xferinfo_callback + * prototype defines. (Deprecates CURLOPT_PROGRESSFUNCTION) */ + CINIT(XFERINFOFUNCTION, FUNCTIONPOINT, 219), + + /* The XOAUTH2 bearer token */ + CINIT(XOAUTH2_BEARER, STRINGPOINT, 220), + + /* Set the interface string to use as outgoing network + * interface for DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_INTERFACE, STRINGPOINT, 221), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP4, STRINGPOINT, 222), + + /* Set the local IPv4 address to use for outgoing DNS requests. + * Only supported by the c-ares DNS backend */ + CINIT(DNS_LOCAL_IP6, STRINGPOINT, 223), + + /* Set authentication options directly */ + CINIT(LOGIN_OPTIONS, STRINGPOINT, 224), + + /* Enable/disable TLS NPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_NPN, LONG, 225), + + /* Enable/disable TLS ALPN extension (http2 over ssl might fail without) */ + CINIT(SSL_ENABLE_ALPN, LONG, 226), + + /* Time to wait for a response to a HTTP request containing an + * Expect: 100-continue header before sending the data anyway. */ + CINIT(EXPECT_100_TIMEOUT_MS, LONG, 227), + + /* This points to a linked list of headers used for proxy requests only, + struct curl_slist kind */ + CINIT(PROXYHEADER, OBJECTPOINT, 228), + + /* Pass in a bitmask of "header options" */ + CINIT(HEADEROPT, LONG, 229), + + /* The public key in DER form used to validate the peer public key + this option is used only if SSL_VERIFYPEER is true */ + CINIT(PINNEDPUBLICKEY, STRINGPOINT, 230), + + /* Path to Unix domain socket */ + CINIT(UNIX_SOCKET_PATH, STRINGPOINT, 231), + + /* Set if we should verify the certificate status. */ + CINIT(SSL_VERIFYSTATUS, LONG, 232), + + /* Set if we should enable TLS false start. */ + CINIT(SSL_FALSESTART, LONG, 233), + + /* Do not squash dot-dot sequences */ + CINIT(PATH_AS_IS, LONG, 234), + + /* Proxy Service Name */ + CINIT(PROXY_SERVICE_NAME, STRINGPOINT, 235), + + /* Service Name */ + CINIT(SERVICE_NAME, STRINGPOINT, 236), + + /* Wait/don't wait for pipe/mutex to clarify */ + CINIT(PIPEWAIT, LONG, 237), + + /* Set the protocol used when curl is given a URL without a protocol */ + CINIT(DEFAULT_PROTOCOL, STRINGPOINT, 238), + + /* Set stream weight, 1 - 256 (default is 16) */ + CINIT(STREAM_WEIGHT, LONG, 239), + + /* Set stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS, OBJECTPOINT, 240), + + /* Set E-xclusive stream dependency on another CURL handle */ + CINIT(STREAM_DEPENDS_E, OBJECTPOINT, 241), + + CURLOPT_LASTENTRY /* the last unused */ +} CURLoption; + +#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all + the obsolete stuff removed! */ + +/* Backwards compatibility with older names */ +/* These are scheduled to disappear by 2011 */ + +/* This was added in version 7.19.1 */ +#define CURLOPT_POST301 CURLOPT_POSTREDIR + +/* These are scheduled to disappear by 2009 */ + +/* The following were added in 7.17.0 */ +#define CURLOPT_SSLKEYPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_FTPAPPEND CURLOPT_APPEND +#define CURLOPT_FTPLISTONLY CURLOPT_DIRLISTONLY +#define CURLOPT_FTP_SSL CURLOPT_USE_SSL + +/* The following were added earlier */ + +#define CURLOPT_SSLCERTPASSWD CURLOPT_KEYPASSWD +#define CURLOPT_KRB4LEVEL CURLOPT_KRBLEVEL + +#else +/* This is set if CURL_NO_OLDIES is defined at compile-time */ +#undef CURLOPT_DNS_USE_GLOBAL_CACHE /* soon obsolete */ +#endif + + + /* Below here follows defines for the CURLOPT_IPRESOLVE option. If a host + name resolves addresses using more than one IP protocol version, this + option might be handy to force libcurl to use a specific IP version. */ +#define CURL_IPRESOLVE_WHATEVER 0 /* default, resolves addresses to all IP + versions that your system allows */ +#define CURL_IPRESOLVE_V4 1 /* resolve to IPv4 addresses */ +#define CURL_IPRESOLVE_V6 2 /* resolve to IPv6 addresses */ + + /* three convenient "aliases" that follow the name scheme better */ +#define CURLOPT_RTSPHEADER CURLOPT_HTTPHEADER + + /* These enums are for use with the CURLOPT_HTTP_VERSION option. */ +enum { + CURL_HTTP_VERSION_NONE, /* setting this means we don't care, and that we'd + like the library to choose the best possible + for us! */ + CURL_HTTP_VERSION_1_0, /* please use HTTP 1.0 in the request */ + CURL_HTTP_VERSION_1_1, /* please use HTTP 1.1 in the request */ + CURL_HTTP_VERSION_2_0, /* please use HTTP 2 in the request */ + CURL_HTTP_VERSION_2TLS, /* use version 2 for HTTPS, version 1.1 for HTTP */ + + CURL_HTTP_VERSION_LAST /* *ILLEGAL* http version */ +}; + +/* Convenience definition simple because the name of the version is HTTP/2 and + not 2.0. The 2_0 version of the enum name was set while the version was + still planned to be 2.0 and we stick to it for compatibility. */ +#define CURL_HTTP_VERSION_2 CURL_HTTP_VERSION_2_0 + +/* + * Public API enums for RTSP requests + */ +enum { + CURL_RTSPREQ_NONE, /* first in list */ + CURL_RTSPREQ_OPTIONS, + CURL_RTSPREQ_DESCRIBE, + CURL_RTSPREQ_ANNOUNCE, + CURL_RTSPREQ_SETUP, + CURL_RTSPREQ_PLAY, + CURL_RTSPREQ_PAUSE, + CURL_RTSPREQ_TEARDOWN, + CURL_RTSPREQ_GET_PARAMETER, + CURL_RTSPREQ_SET_PARAMETER, + CURL_RTSPREQ_RECORD, + CURL_RTSPREQ_RECEIVE, + CURL_RTSPREQ_LAST /* last in list */ +}; + + /* These enums are for use with the CURLOPT_NETRC option. */ +enum CURL_NETRC_OPTION { + CURL_NETRC_IGNORED, /* The .netrc will never be read. + * This is the default. */ + CURL_NETRC_OPTIONAL, /* A user:password in the URL will be preferred + * to one in the .netrc. */ + CURL_NETRC_REQUIRED, /* A user:password in the URL will be ignored. + * Unless one is set programmatically, the .netrc + * will be queried. */ + CURL_NETRC_LAST +}; + +enum { + CURL_SSLVERSION_DEFAULT, + CURL_SSLVERSION_TLSv1, /* TLS 1.x */ + CURL_SSLVERSION_SSLv2, + CURL_SSLVERSION_SSLv3, + CURL_SSLVERSION_TLSv1_0, + CURL_SSLVERSION_TLSv1_1, + CURL_SSLVERSION_TLSv1_2, + + CURL_SSLVERSION_LAST /* never use, keep last */ +}; + +enum CURL_TLSAUTH { + CURL_TLSAUTH_NONE, + CURL_TLSAUTH_SRP, + CURL_TLSAUTH_LAST /* never use, keep last */ +}; + +/* symbols to use with CURLOPT_POSTREDIR. + CURL_REDIR_POST_301, CURL_REDIR_POST_302 and CURL_REDIR_POST_303 + can be bitwise ORed so that CURL_REDIR_POST_301 | CURL_REDIR_POST_302 + | CURL_REDIR_POST_303 == CURL_REDIR_POST_ALL */ + +#define CURL_REDIR_GET_ALL 0 +#define CURL_REDIR_POST_301 1 +#define CURL_REDIR_POST_302 2 +#define CURL_REDIR_POST_303 4 +#define CURL_REDIR_POST_ALL \ + (CURL_REDIR_POST_301|CURL_REDIR_POST_302|CURL_REDIR_POST_303) + +typedef enum { + CURL_TIMECOND_NONE, + + CURL_TIMECOND_IFMODSINCE, + CURL_TIMECOND_IFUNMODSINCE, + CURL_TIMECOND_LASTMOD, + + CURL_TIMECOND_LAST +} curl_TimeCond; + + +/* curl_strequal() and curl_strnequal() are subject for removal in a future + libcurl, see lib/README.curlx for details */ +CURL_EXTERN int (curl_strequal)(const char *s1, const char *s2); +CURL_EXTERN int (curl_strnequal)(const char *s1, const char *s2, size_t n); + +/* name is uppercase CURLFORM_ */ +#ifdef CFINIT +#undef CFINIT +#endif + +#ifdef CURL_ISOCPP +#define CFINIT(name) CURLFORM_ ## name +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define CFINIT(name) CURLFORM_/**/name +#endif + +typedef enum { + CFINIT(NOTHING), /********* the first one is unused ************/ + + /* */ + CFINIT(COPYNAME), + CFINIT(PTRNAME), + CFINIT(NAMELENGTH), + CFINIT(COPYCONTENTS), + CFINIT(PTRCONTENTS), + CFINIT(CONTENTSLENGTH), + CFINIT(FILECONTENT), + CFINIT(ARRAY), + CFINIT(OBSOLETE), + CFINIT(FILE), + + CFINIT(BUFFER), + CFINIT(BUFFERPTR), + CFINIT(BUFFERLENGTH), + + CFINIT(CONTENTTYPE), + CFINIT(CONTENTHEADER), + CFINIT(FILENAME), + CFINIT(END), + CFINIT(OBSOLETE2), + + CFINIT(STREAM), + CFINIT(CONTENTLEN), /* added in 7.46.0, provide a curl_off_t length */ + + CURLFORM_LASTENTRY /* the last unused */ +} CURLformoption; + +#undef CFINIT /* done */ + +/* structure to be used as parameter for CURLFORM_ARRAY */ +struct curl_forms { + CURLformoption option; + const char *value; +}; + +/* use this for multipart formpost building */ +/* Returns code for curl_formadd() + * + * Returns: + * CURL_FORMADD_OK on success + * CURL_FORMADD_MEMORY if the FormInfo allocation fails + * CURL_FORMADD_OPTION_TWICE if one option is given twice for one Form + * CURL_FORMADD_NULL if a null pointer was given for a char + * CURL_FORMADD_MEMORY if the allocation of a FormInfo struct failed + * CURL_FORMADD_UNKNOWN_OPTION if an unknown option was used + * CURL_FORMADD_INCOMPLETE if the some FormInfo is not complete (or error) + * CURL_FORMADD_MEMORY if a curl_httppost struct cannot be allocated + * CURL_FORMADD_MEMORY if some allocation for string copying failed. + * CURL_FORMADD_ILLEGAL_ARRAY if an illegal option is used in an array + * + ***************************************************************************/ +typedef enum { + CURL_FORMADD_OK, /* first, no error */ + + CURL_FORMADD_MEMORY, + CURL_FORMADD_OPTION_TWICE, + CURL_FORMADD_NULL, + CURL_FORMADD_UNKNOWN_OPTION, + CURL_FORMADD_INCOMPLETE, + CURL_FORMADD_ILLEGAL_ARRAY, + CURL_FORMADD_DISABLED, /* libcurl was built with this disabled */ + + CURL_FORMADD_LAST /* last */ +} CURLFORMcode; + +/* + * NAME curl_formadd() + * + * DESCRIPTION + * + * Pretty advanced function for building multi-part formposts. Each invoke + * adds one part that together construct a full post. Then use + * CURLOPT_HTTPPOST to send it off to libcurl. + */ +CURL_EXTERN CURLFORMcode curl_formadd(struct curl_httppost **httppost, + struct curl_httppost **last_post, + ...); + +/* + * callback function for curl_formget() + * The void *arg pointer will be the one passed as second argument to + * curl_formget(). + * The character buffer passed to it must not be freed. + * Should return the buffer length passed to it as the argument "len" on + * success. + */ +typedef size_t (*curl_formget_callback)(void *arg, const char *buf, + size_t len); + +/* + * NAME curl_formget() + * + * DESCRIPTION + * + * Serialize a curl_httppost struct built with curl_formadd(). + * Accepts a void pointer as second argument which will be passed to + * the curl_formget_callback function. + * Returns 0 on success. + */ +CURL_EXTERN int curl_formget(struct curl_httppost *form, void *arg, + curl_formget_callback append); +/* + * NAME curl_formfree() + * + * DESCRIPTION + * + * Free a multipart formpost previously built with curl_formadd(). + */ +CURL_EXTERN void curl_formfree(struct curl_httppost *form); + +/* + * NAME curl_getenv() + * + * DESCRIPTION + * + * Returns a malloc()'ed string that MUST be curl_free()ed after usage is + * complete. DEPRECATED - see lib/README.curlx + */ +CURL_EXTERN char *curl_getenv(const char *variable); + +/* + * NAME curl_version() + * + * DESCRIPTION + * + * Returns a static ascii string of the libcurl version. + */ +CURL_EXTERN char *curl_version(void); + +/* + * NAME curl_easy_escape() + * + * DESCRIPTION + * + * Escapes URL strings (converts all letters consider illegal in URLs to their + * %XX versions). This function returns a new allocated string or NULL if an + * error occurred. + */ +CURL_EXTERN char *curl_easy_escape(CURL *handle, + const char *string, + int length); + +/* the previous version: */ +CURL_EXTERN char *curl_escape(const char *string, + int length); + + +/* + * NAME curl_easy_unescape() + * + * DESCRIPTION + * + * Unescapes URL encoding in strings (converts all %XX codes to their 8bit + * versions). This function returns a new allocated string or NULL if an error + * occurred. + * Conversion Note: On non-ASCII platforms the ASCII %XX codes are + * converted into the host encoding. + */ +CURL_EXTERN char *curl_easy_unescape(CURL *handle, + const char *string, + int length, + int *outlength); + +/* the previous version */ +CURL_EXTERN char *curl_unescape(const char *string, + int length); + +/* + * NAME curl_free() + * + * DESCRIPTION + * + * Provided for de-allocation in the same translation unit that did the + * allocation. Added in libcurl 7.10 + */ +CURL_EXTERN void curl_free(void *p); + +/* + * NAME curl_global_init() + * + * DESCRIPTION + * + * curl_global_init() should be invoked exactly once for each application that + * uses libcurl and before any call of other libcurl functions. + * + * This function is not thread-safe! + */ +CURL_EXTERN CURLcode curl_global_init(long flags); + +/* + * NAME curl_global_init_mem() + * + * DESCRIPTION + * + * curl_global_init() or curl_global_init_mem() should be invoked exactly once + * for each application that uses libcurl. This function can be used to + * initialize libcurl and set user defined memory management callback + * functions. Users can implement memory management routines to check for + * memory leaks, check for mis-use of the curl library etc. User registered + * callback routines with be invoked by this library instead of the system + * memory management routines like malloc, free etc. + */ +CURL_EXTERN CURLcode curl_global_init_mem(long flags, + curl_malloc_callback m, + curl_free_callback f, + curl_realloc_callback r, + curl_strdup_callback s, + curl_calloc_callback c); + +/* + * NAME curl_global_cleanup() + * + * DESCRIPTION + * + * curl_global_cleanup() should be invoked exactly once for each application + * that uses libcurl + */ +CURL_EXTERN void curl_global_cleanup(void); + +/* linked-list structure for the CURLOPT_QUOTE option (and other) */ +struct curl_slist { + char *data; + struct curl_slist *next; +}; + +/* + * NAME curl_slist_append() + * + * DESCRIPTION + * + * Appends a string to a linked list. If no list exists, it will be created + * first. Returns the new list, after appending. + */ +CURL_EXTERN struct curl_slist *curl_slist_append(struct curl_slist *, + const char *); + +/* + * NAME curl_slist_free_all() + * + * DESCRIPTION + * + * free a previously built curl_slist. + */ +CURL_EXTERN void curl_slist_free_all(struct curl_slist *); + +/* + * NAME curl_getdate() + * + * DESCRIPTION + * + * Returns the time, in seconds since 1 Jan 1970 of the time string given in + * the first argument. The time argument in the second parameter is unused + * and should be set to NULL. + */ +CURL_EXTERN time_t curl_getdate(const char *p, const time_t *unused); + +/* info about the certificate chain, only for OpenSSL builds. Asked + for with CURLOPT_CERTINFO / CURLINFO_CERTINFO */ +struct curl_certinfo { + int num_of_certs; /* number of certificates with information */ + struct curl_slist **certinfo; /* for each index in this array, there's a + linked list with textual information in the + format "name: value" */ +}; + +/* enum for the different supported SSL backends */ +typedef enum { + CURLSSLBACKEND_NONE = 0, + CURLSSLBACKEND_OPENSSL = 1, + CURLSSLBACKEND_GNUTLS = 2, + CURLSSLBACKEND_NSS = 3, + CURLSSLBACKEND_OBSOLETE4 = 4, /* Was QSOSSL. */ + CURLSSLBACKEND_GSKIT = 5, + CURLSSLBACKEND_POLARSSL = 6, + CURLSSLBACKEND_CYASSL = 7, + CURLSSLBACKEND_SCHANNEL = 8, + CURLSSLBACKEND_DARWINSSL = 9, + CURLSSLBACKEND_AXTLS = 10, + CURLSSLBACKEND_MBEDTLS = 11 +} curl_sslbackend; + +/* Information about the SSL library used and the respective internal SSL + handle, which can be used to obtain further information regarding the + connection. Asked for with CURLINFO_TLS_SESSION. */ +struct curl_tlssessioninfo { + curl_sslbackend backend; + void *internals; +}; + +#define CURLINFO_STRING 0x100000 +#define CURLINFO_LONG 0x200000 +#define CURLINFO_DOUBLE 0x300000 +#define CURLINFO_SLIST 0x400000 +#define CURLINFO_SOCKET 0x500000 +#define CURLINFO_MASK 0x0fffff +#define CURLINFO_TYPEMASK 0xf00000 + +typedef enum { + CURLINFO_NONE, /* first, never use this */ + CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1, + CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2, + CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3, + CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4, + CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5, + CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6, + CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7, + CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8, + CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9, + CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10, + CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11, + CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12, + CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13, + CURLINFO_FILETIME = CURLINFO_LONG + 14, + CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15, + CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16, + CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17, + CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18, + CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19, + CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20, + CURLINFO_PRIVATE = CURLINFO_STRING + 21, + CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22, + CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23, + CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24, + CURLINFO_OS_ERRNO = CURLINFO_LONG + 25, + CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26, + CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27, + CURLINFO_COOKIELIST = CURLINFO_SLIST + 28, + CURLINFO_LASTSOCKET = CURLINFO_LONG + 29, + CURLINFO_FTP_ENTRY_PATH = CURLINFO_STRING + 30, + CURLINFO_REDIRECT_URL = CURLINFO_STRING + 31, + CURLINFO_PRIMARY_IP = CURLINFO_STRING + 32, + CURLINFO_APPCONNECT_TIME = CURLINFO_DOUBLE + 33, + CURLINFO_CERTINFO = CURLINFO_SLIST + 34, + CURLINFO_CONDITION_UNMET = CURLINFO_LONG + 35, + CURLINFO_RTSP_SESSION_ID = CURLINFO_STRING + 36, + CURLINFO_RTSP_CLIENT_CSEQ = CURLINFO_LONG + 37, + CURLINFO_RTSP_SERVER_CSEQ = CURLINFO_LONG + 38, + CURLINFO_RTSP_CSEQ_RECV = CURLINFO_LONG + 39, + CURLINFO_PRIMARY_PORT = CURLINFO_LONG + 40, + CURLINFO_LOCAL_IP = CURLINFO_STRING + 41, + CURLINFO_LOCAL_PORT = CURLINFO_LONG + 42, + CURLINFO_TLS_SESSION = CURLINFO_SLIST + 43, + CURLINFO_ACTIVESOCKET = CURLINFO_SOCKET + 44, + /* Fill in new entries below here! */ + + CURLINFO_LASTONE = 44 +} CURLINFO; + +/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as + CURLINFO_HTTP_CODE */ +#define CURLINFO_HTTP_CODE CURLINFO_RESPONSE_CODE + +typedef enum { + CURLCLOSEPOLICY_NONE, /* first, never use this */ + + CURLCLOSEPOLICY_OLDEST, + CURLCLOSEPOLICY_LEAST_RECENTLY_USED, + CURLCLOSEPOLICY_LEAST_TRAFFIC, + CURLCLOSEPOLICY_SLOWEST, + CURLCLOSEPOLICY_CALLBACK, + + CURLCLOSEPOLICY_LAST /* last, never use this */ +} curl_closepolicy; + +#define CURL_GLOBAL_SSL (1<<0) +#define CURL_GLOBAL_WIN32 (1<<1) +#define CURL_GLOBAL_ALL (CURL_GLOBAL_SSL|CURL_GLOBAL_WIN32) +#define CURL_GLOBAL_NOTHING 0 +#define CURL_GLOBAL_DEFAULT CURL_GLOBAL_ALL +#define CURL_GLOBAL_ACK_EINTR (1<<2) + + +/***************************************************************************** + * Setup defines, protos etc for the sharing stuff. + */ + +/* Different data locks for a single share */ +typedef enum { + CURL_LOCK_DATA_NONE = 0, + /* CURL_LOCK_DATA_SHARE is used internally to say that + * the locking is just made to change the internal state of the share + * itself. + */ + CURL_LOCK_DATA_SHARE, + CURL_LOCK_DATA_COOKIE, + CURL_LOCK_DATA_DNS, + CURL_LOCK_DATA_SSL_SESSION, + CURL_LOCK_DATA_CONNECT, + CURL_LOCK_DATA_LAST +} curl_lock_data; + +/* Different lock access types */ +typedef enum { + CURL_LOCK_ACCESS_NONE = 0, /* unspecified action */ + CURL_LOCK_ACCESS_SHARED = 1, /* for read perhaps */ + CURL_LOCK_ACCESS_SINGLE = 2, /* for write perhaps */ + CURL_LOCK_ACCESS_LAST /* never use */ +} curl_lock_access; + +typedef void (*curl_lock_function)(CURL *handle, + curl_lock_data data, + curl_lock_access locktype, + void *userptr); +typedef void (*curl_unlock_function)(CURL *handle, + curl_lock_data data, + void *userptr); + +typedef void CURLSH; + +typedef enum { + CURLSHE_OK, /* all is fine */ + CURLSHE_BAD_OPTION, /* 1 */ + CURLSHE_IN_USE, /* 2 */ + CURLSHE_INVALID, /* 3 */ + CURLSHE_NOMEM, /* 4 out of memory */ + CURLSHE_NOT_BUILT_IN, /* 5 feature not present in lib */ + CURLSHE_LAST /* never use */ +} CURLSHcode; + +typedef enum { + CURLSHOPT_NONE, /* don't use */ + CURLSHOPT_SHARE, /* specify a data type to share */ + CURLSHOPT_UNSHARE, /* specify which data type to stop sharing */ + CURLSHOPT_LOCKFUNC, /* pass in a 'curl_lock_function' pointer */ + CURLSHOPT_UNLOCKFUNC, /* pass in a 'curl_unlock_function' pointer */ + CURLSHOPT_USERDATA, /* pass in a user data pointer used in the lock/unlock + callback functions */ + CURLSHOPT_LAST /* never use */ +} CURLSHoption; + +CURL_EXTERN CURLSH *curl_share_init(void); +CURL_EXTERN CURLSHcode curl_share_setopt(CURLSH *, CURLSHoption option, ...); +CURL_EXTERN CURLSHcode curl_share_cleanup(CURLSH *); + +/**************************************************************************** + * Structures for querying information about the curl library at runtime. + */ + +typedef enum { + CURLVERSION_FIRST, + CURLVERSION_SECOND, + CURLVERSION_THIRD, + CURLVERSION_FOURTH, + CURLVERSION_LAST /* never actually use this */ +} CURLversion; + +/* The 'CURLVERSION_NOW' is the symbolic name meant to be used by + basically all programs ever that want to get version information. It is + meant to be a built-in version number for what kind of struct the caller + expects. If the struct ever changes, we redefine the NOW to another enum + from above. */ +#define CURLVERSION_NOW CURLVERSION_FOURTH + +typedef struct { + CURLversion age; /* age of the returned struct */ + const char *version; /* LIBCURL_VERSION */ + unsigned int version_num; /* LIBCURL_VERSION_NUM */ + const char *host; /* OS/host/cpu/machine when configured */ + int features; /* bitmask, see defines below */ + const char *ssl_version; /* human readable string */ + long ssl_version_num; /* not used anymore, always 0 */ + const char *libz_version; /* human readable string */ + /* protocols is terminated by an entry with a NULL protoname */ + const char * const *protocols; + + /* The fields below this were added in CURLVERSION_SECOND */ + const char *ares; + int ares_num; + + /* This field was added in CURLVERSION_THIRD */ + const char *libidn; + + /* These field were added in CURLVERSION_FOURTH */ + + /* Same as '_libiconv_version' if built with HAVE_ICONV */ + int iconv_ver_num; + + const char *libssh_version; /* human readable string */ + +} curl_version_info_data; + +#define CURL_VERSION_IPV6 (1<<0) /* IPv6-enabled */ +#define CURL_VERSION_KERBEROS4 (1<<1) /* Kerberos V4 auth is supported + (deprecated) */ +#define CURL_VERSION_SSL (1<<2) /* SSL options are present */ +#define CURL_VERSION_LIBZ (1<<3) /* libz features are present */ +#define CURL_VERSION_NTLM (1<<4) /* NTLM auth is supported */ +#define CURL_VERSION_GSSNEGOTIATE (1<<5) /* Negotiate auth is supported + (deprecated) */ +#define CURL_VERSION_DEBUG (1<<6) /* Built with debug capabilities */ +#define CURL_VERSION_ASYNCHDNS (1<<7) /* Asynchronous DNS resolves */ +#define CURL_VERSION_SPNEGO (1<<8) /* SPNEGO auth is supported */ +#define CURL_VERSION_LARGEFILE (1<<9) /* Supports files larger than 2GB */ +#define CURL_VERSION_IDN (1<<10) /* Internationized Domain Names are + supported */ +#define CURL_VERSION_SSPI (1<<11) /* Built against Windows SSPI */ +#define CURL_VERSION_CONV (1<<12) /* Character conversions supported */ +#define CURL_VERSION_CURLDEBUG (1<<13) /* Debug memory tracking supported */ +#define CURL_VERSION_TLSAUTH_SRP (1<<14) /* TLS-SRP auth is supported */ +#define CURL_VERSION_NTLM_WB (1<<15) /* NTLM delegation to winbind helper + is suported */ +#define CURL_VERSION_HTTP2 (1<<16) /* HTTP2 support built-in */ +#define CURL_VERSION_GSSAPI (1<<17) /* Built against a GSS-API library */ +#define CURL_VERSION_KERBEROS5 (1<<18) /* Kerberos V5 auth is supported */ +#define CURL_VERSION_UNIX_SOCKETS (1<<19) /* Unix domain sockets support */ +#define CURL_VERSION_PSL (1<<20) /* Mozilla's Public Suffix List, used + for cookie domain verification */ + + /* + * NAME curl_version_info() + * + * DESCRIPTION + * + * This function returns a pointer to a static copy of the version info + * struct. See above. + */ +CURL_EXTERN curl_version_info_data *curl_version_info(CURLversion); + +/* + * NAME curl_easy_strerror() + * + * DESCRIPTION + * + * The curl_easy_strerror function may be used to turn a CURLcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_easy_strerror(CURLcode); + +/* + * NAME curl_share_strerror() + * + * DESCRIPTION + * + * The curl_share_strerror function may be used to turn a CURLSHcode value + * into the equivalent human readable error string. This is useful + * for printing meaningful error messages. + */ +CURL_EXTERN const char *curl_share_strerror(CURLSHcode); + +/* + * NAME curl_easy_pause() + * + * DESCRIPTION + * + * The curl_easy_pause function pauses or unpauses transfers. Select the new + * state by setting the bitmask, use the convenience defines below. + * + */ +CURL_EXTERN CURLcode curl_easy_pause(CURL *handle, int bitmask); + +#define CURLPAUSE_RECV (1<<0) +#define CURLPAUSE_RECV_CONT (0) + +#define CURLPAUSE_SEND (1<<2) +#define CURLPAUSE_SEND_CONT (0) + +#define CURLPAUSE_ALL (CURLPAUSE_RECV|CURLPAUSE_SEND) +#define CURLPAUSE_CONT (CURLPAUSE_RECV_CONT|CURLPAUSE_SEND_CONT) + +#ifdef __cplusplus +} +#endif + +/* unfortunately, the easy.h and multi.h include files need options and info + stuff before they can be included! */ +#include "easy.h" /* nothing in curl is fun without the easy stuff */ +#include "multi.h" + +/* the typechecker doesn't work in C++ (yet) */ +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && \ + !defined(__cplusplus) && !defined(CURL_DISABLE_TYPECHECK) +#include "typecheck-gcc.h" +#else +#if defined(__STDC__) && (__STDC__ >= 1) +/* This preprocessor magic that replaces a call with the exact same call is + only done to make sure application authors pass exactly three arguments + to these functions. */ +#define curl_easy_setopt(handle,opt,param) curl_easy_setopt(handle,opt,param) +#define curl_easy_getinfo(handle,info,arg) curl_easy_getinfo(handle,info,arg) +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) +#endif /* __STDC__ >= 1 */ +#endif /* gcc >= 4.3 && !__cplusplus */ + +#endif /* __CURL_CURL_H */ diff --git a/deps/libcurl/include/curl/curlbuild.h b/deps/libcurl/include/curl/curlbuild.h new file mode 100644 index 0000000000000000000000000000000000000000..bdca52b5355feec5f80f279bea6c5799bcf1041e --- /dev/null +++ b/deps/libcurl/include/curl/curlbuild.h @@ -0,0 +1,198 @@ +/* include/curl/curlbuild.h. Generated from curlbuild.h.in by configure. */ +#ifndef __CURL_CURLBUILD_H +#define __CURL_CURLBUILD_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* ================================================================ */ +/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */ +/* ================================================================ */ + +/* + * NOTE 1: + * ------- + * + * Nothing in this file is intended to be modified or adjusted by the + * curl library user nor by the curl library builder. + * + * If you think that something actually needs to be changed, adjusted + * or fixed in this file, then, report it on the libcurl development + * mailing list: http://cool.haxx.se/mailman/listinfo/curl-library/ + * + * This header file shall only export symbols which are 'curl' or 'CURL' + * prefixed, otherwise public name space would be polluted. + * + * NOTE 2: + * ------- + * + * Right now you might be staring at file include/curl/curlbuild.h.in or + * at file include/curl/curlbuild.h, this is due to the following reason: + * + * On systems capable of running the configure script, the configure process + * will overwrite the distributed include/curl/curlbuild.h file with one that + * is suitable and specific to the library being configured and built, which + * is generated from the include/curl/curlbuild.h.in template file. + * + */ + +/* ================================================================ */ +/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */ +/* ================================================================ */ + +#ifdef CURL_SIZEOF_LONG +#error "CURL_SIZEOF_LONG shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SIZEOF_LONG_already_defined +#endif + +#ifdef CURL_TYPEOF_CURL_SOCKLEN_T +#error "CURL_TYPEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_already_defined +#endif + +#ifdef CURL_SIZEOF_CURL_SOCKLEN_T +#error "CURL_SIZEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_already_defined +#endif + +#ifdef CURL_TYPEOF_CURL_OFF_T +#error "CURL_TYPEOF_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_FORMAT_CURL_OFF_T +#error "CURL_FORMAT_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_FORMAT_CURL_OFF_TU +#error "CURL_FORMAT_CURL_OFF_TU shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_already_defined +#endif + +#ifdef CURL_FORMAT_OFF_T +#error "CURL_FORMAT_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_FORMAT_OFF_T_already_defined +#endif + +#ifdef CURL_SIZEOF_CURL_OFF_T +#error "CURL_SIZEOF_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_SUFFIX_CURL_OFF_T +#error "CURL_SUFFIX_CURL_OFF_T shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_already_defined +#endif + +#ifdef CURL_SUFFIX_CURL_OFF_TU +#error "CURL_SUFFIX_CURL_OFF_TU shall not be defined except in curlbuild.h" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_already_defined +#endif + +/* ================================================================ */ +/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */ +/* ================================================================ */ + +/* Configure process defines this to 1 when it finds out that system */ +/* header file ws2tcpip.h must be included by the external interface. */ +/* #undef CURL_PULL_WS2TCPIP_H */ +#ifdef CURL_PULL_WS2TCPIP_H +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/types.h must be included by the external interface. */ +#define CURL_PULL_SYS_TYPES_H 1 +#ifdef CURL_PULL_SYS_TYPES_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file stdint.h must be included by the external interface. */ +/* #undef CURL_PULL_STDINT_H */ +#ifdef CURL_PULL_STDINT_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file inttypes.h must be included by the external interface. */ +/* #undef CURL_PULL_INTTYPES_H */ +#ifdef CURL_PULL_INTTYPES_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/socket.h must be included by the external interface. */ +#define CURL_PULL_SYS_SOCKET_H 1 +#ifdef CURL_PULL_SYS_SOCKET_H +# include +#endif + +/* Configure process defines this to 1 when it finds out that system */ +/* header file sys/poll.h must be included by the external interface. */ +/* #undef CURL_PULL_SYS_POLL_H */ +#ifdef CURL_PULL_SYS_POLL_H +# include +#endif + +/* The size of `long', as computed by sizeof. */ +#define CURL_SIZEOF_LONG 8 + +/* Integral data type used for curl_socklen_t. */ +#define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t + +/* The size of `curl_socklen_t', as computed by sizeof. */ +#define CURL_SIZEOF_CURL_SOCKLEN_T 4 + +/* Data type definition of curl_socklen_t. */ +typedef CURL_TYPEOF_CURL_SOCKLEN_T curl_socklen_t; + +/* Signed integral data type used for curl_off_t. */ +#define CURL_TYPEOF_CURL_OFF_T long + +/* Data type definition of curl_off_t. */ +typedef CURL_TYPEOF_CURL_OFF_T curl_off_t; + +/* curl_off_t formatting string directive without "%" conversion specifier. */ +#define CURL_FORMAT_CURL_OFF_T "ld" + +/* unsigned curl_off_t formatting string without "%" conversion specifier. */ +#define CURL_FORMAT_CURL_OFF_TU "lu" + +/* curl_off_t formatting string directive with "%" conversion specifier. */ +#define CURL_FORMAT_OFF_T "%ld" + +/* The size of `curl_off_t', as computed by sizeof. */ +#define CURL_SIZEOF_CURL_OFF_T 8 + +/* curl_off_t constant suffix. */ +#define CURL_SUFFIX_CURL_OFF_T L + +/* unsigned curl_off_t constant suffix. */ +#define CURL_SUFFIX_CURL_OFF_TU UL + +#endif /* __CURL_CURLBUILD_H */ diff --git a/deps/libcurl/include/curl/curlrules.h b/deps/libcurl/include/curl/curlrules.h new file mode 100644 index 0000000000000000000000000000000000000000..7c2ede35b63a7540acd042ce5c2b27a70c982d02 --- /dev/null +++ b/deps/libcurl/include/curl/curlrules.h @@ -0,0 +1,262 @@ +#ifndef __CURL_CURLRULES_H +#define __CURL_CURLRULES_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* ================================================================ */ +/* COMPILE TIME SANITY CHECKS */ +/* ================================================================ */ + +/* + * NOTE 1: + * ------- + * + * All checks done in this file are intentionally placed in a public + * header file which is pulled by curl/curl.h when an application is + * being built using an already built libcurl library. Additionally + * this file is also included and used when building the library. + * + * If compilation fails on this file it is certainly sure that the + * problem is elsewhere. It could be a problem in the curlbuild.h + * header file, or simply that you are using different compilation + * settings than those used to build the library. + * + * Nothing in this file is intended to be modified or adjusted by the + * curl library user nor by the curl library builder. + * + * Do not deactivate any check, these are done to make sure that the + * library is properly built and used. + * + * You can find further help on the libcurl development mailing list: + * http://cool.haxx.se/mailman/listinfo/curl-library/ + * + * NOTE 2 + * ------ + * + * Some of the following compile time checks are based on the fact + * that the dimension of a constant array can not be a negative one. + * In this way if the compile time verification fails, the compilation + * will fail issuing an error. The error description wording is compiler + * dependent but it will be quite similar to one of the following: + * + * "negative subscript or subscript is too large" + * "array must have at least one element" + * "-1 is an illegal array size" + * "size of array is negative" + * + * If you are building an application which tries to use an already + * built libcurl library and you are getting this kind of errors on + * this file, it is a clear indication that there is a mismatch between + * how the library was built and how you are trying to use it for your + * application. Your already compiled or binary library provider is the + * only one who can give you the details you need to properly use it. + */ + +/* + * Verify that some macros are actually defined. + */ + +#ifndef CURL_SIZEOF_LONG +# error "CURL_SIZEOF_LONG definition is missing!" + Error Compilation_aborted_CURL_SIZEOF_LONG_is_missing +#endif + +#ifndef CURL_TYPEOF_CURL_SOCKLEN_T +# error "CURL_TYPEOF_CURL_SOCKLEN_T definition is missing!" + Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_is_missing +#endif + +#ifndef CURL_SIZEOF_CURL_SOCKLEN_T +# error "CURL_SIZEOF_CURL_SOCKLEN_T definition is missing!" + Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_is_missing +#endif + +#ifndef CURL_TYPEOF_CURL_OFF_T +# error "CURL_TYPEOF_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_FORMAT_CURL_OFF_T +# error "CURL_FORMAT_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_FORMAT_CURL_OFF_TU +# error "CURL_FORMAT_CURL_OFF_TU definition is missing!" + Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_is_missing +#endif + +#ifndef CURL_FORMAT_OFF_T +# error "CURL_FORMAT_OFF_T definition is missing!" + Error Compilation_aborted_CURL_FORMAT_OFF_T_is_missing +#endif + +#ifndef CURL_SIZEOF_CURL_OFF_T +# error "CURL_SIZEOF_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_SUFFIX_CURL_OFF_T +# error "CURL_SUFFIX_CURL_OFF_T definition is missing!" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_is_missing +#endif + +#ifndef CURL_SUFFIX_CURL_OFF_TU +# error "CURL_SUFFIX_CURL_OFF_TU definition is missing!" + Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_is_missing +#endif + +/* + * Macros private to this header file. + */ + +#define CurlchkszEQ(t, s) sizeof(t) == s ? 1 : -1 + +#define CurlchkszGE(t1, t2) sizeof(t1) >= sizeof(t2) ? 1 : -1 + +/* + * Verify that the size previously defined and expected for long + * is the same as the one reported by sizeof() at compile time. + */ + +typedef char + __curl_rule_01__ + [CurlchkszEQ(long, CURL_SIZEOF_LONG)]; + +/* + * Verify that the size previously defined and expected for + * curl_off_t is actually the the same as the one reported + * by sizeof() at compile time. + */ + +typedef char + __curl_rule_02__ + [CurlchkszEQ(curl_off_t, CURL_SIZEOF_CURL_OFF_T)]; + +/* + * Verify at compile time that the size of curl_off_t as reported + * by sizeof() is greater or equal than the one reported for long + * for the current compilation. + */ + +typedef char + __curl_rule_03__ + [CurlchkszGE(curl_off_t, long)]; + +/* + * Verify that the size previously defined and expected for + * curl_socklen_t is actually the the same as the one reported + * by sizeof() at compile time. + */ + +typedef char + __curl_rule_04__ + [CurlchkszEQ(curl_socklen_t, CURL_SIZEOF_CURL_SOCKLEN_T)]; + +/* + * Verify at compile time that the size of curl_socklen_t as reported + * by sizeof() is greater or equal than the one reported for int for + * the current compilation. + */ + +typedef char + __curl_rule_05__ + [CurlchkszGE(curl_socklen_t, int)]; + +/* ================================================================ */ +/* EXTERNALLY AND INTERNALLY VISIBLE DEFINITIONS */ +/* ================================================================ */ + +/* + * CURL_ISOCPP and CURL_OFF_T_C definitions are done here in order to allow + * these to be visible and exported by the external libcurl interface API, + * while also making them visible to the library internals, simply including + * curl_setup.h, without actually needing to include curl.h internally. + * If some day this section would grow big enough, all this should be moved + * to its own header file. + */ + +/* + * Figure out if we can use the ## preprocessor operator, which is supported + * by ISO/ANSI C and C++. Some compilers support it without setting __STDC__ + * or __cplusplus so we need to carefully check for them too. + */ + +#if defined(__STDC__) || defined(_MSC_VER) || defined(__cplusplus) || \ + defined(__HP_aCC) || defined(__BORLANDC__) || defined(__LCC__) || \ + defined(__POCC__) || defined(__SALFORDC__) || defined(__HIGHC__) || \ + defined(__ILEC400__) + /* This compiler is believed to have an ISO compatible preprocessor */ +#define CURL_ISOCPP +#else + /* This compiler is believed NOT to have an ISO compatible preprocessor */ +#undef CURL_ISOCPP +#endif + +/* + * Macros for minimum-width signed and unsigned curl_off_t integer constants. + */ + +#if defined(__BORLANDC__) && (__BORLANDC__ == 0x0551) +# define __CURL_OFF_T_C_HLPR2(x) x +# define __CURL_OFF_T_C_HLPR1(x) __CURL_OFF_T_C_HLPR2(x) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ + __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_TU) +#else +# ifdef CURL_ISOCPP +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val ## Suffix +# else +# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val/**/Suffix +# endif +# define __CURL_OFF_T_C_HLPR1(Val,Suffix) __CURL_OFF_T_C_HLPR2(Val,Suffix) +# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_T) +# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_TU) +#endif + +/* + * Get rid of macros private to this header file. + */ + +#undef CurlchkszEQ +#undef CurlchkszGE + +/* + * Get rid of macros not intended to exist beyond this point. + */ + +#undef CURL_PULL_WS2TCPIP_H +#undef CURL_PULL_SYS_TYPES_H +#undef CURL_PULL_SYS_SOCKET_H +#undef CURL_PULL_SYS_POLL_H +#undef CURL_PULL_STDINT_H +#undef CURL_PULL_INTTYPES_H + +#undef CURL_TYPEOF_CURL_SOCKLEN_T +#undef CURL_TYPEOF_CURL_OFF_T + +#ifdef CURL_NO_OLDIES +#undef CURL_FORMAT_OFF_T /* not required since 7.19.0 - obsoleted in 7.20.0 */ +#endif + +#endif /* __CURL_CURLRULES_H */ diff --git a/deps/libcurl/include/curl/curlver.h b/deps/libcurl/include/curl/curlver.h new file mode 100644 index 0000000000000000000000000000000000000000..17906764c898e69ce28204bda7d74573f4c365ee --- /dev/null +++ b/deps/libcurl/include/curl/curlver.h @@ -0,0 +1,77 @@ +#ifndef __CURL_CURLVER_H +#define __CURL_CURLVER_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* This header file contains nothing but libcurl version info, generated by + a script at release-time. This was made its own header file in 7.11.2 */ + +/* This is the global package copyright */ +#define LIBCURL_COPYRIGHT "1996 - 2015 Daniel Stenberg, ." + +/* This is the version number of the libcurl package from which this header + file origins: */ +#define LIBCURL_VERSION "7.47.0" + +/* The numeric version number is also available "in parts" by using these + defines: */ +#define LIBCURL_VERSION_MAJOR 7 +#define LIBCURL_VERSION_MINOR 47 +#define LIBCURL_VERSION_PATCH 0 + +/* This is the numeric version of the libcurl version number, meant for easier + parsing and comparions by programs. The LIBCURL_VERSION_NUM define will + always follow this syntax: + + 0xXXYYZZ + + Where XX, YY and ZZ are the main version, release and patch numbers in + hexadecimal (using 8 bits each). All three numbers are always represented + using two digits. 1.2 would appear as "0x010200" while version 9.11.7 + appears as "0x090b07". + + This 6-digit (24 bits) hexadecimal number does not show pre-release number, + and it is always a greater number in a more recent release. It makes + comparisons with greater than and less than work. + + Note: This define is the full hex number and _does not_ use the + CURL_VERSION_BITS() macro since curl's own configure script greps for it + and needs it to contain the full number. +*/ +#define LIBCURL_VERSION_NUM 0x072f00 + +/* + * This is the date and time when the full source package was created. The + * timestamp is not stored in git, as the timestamp is properly set in the + * tarballs by the maketgz script. + * + * The format of the date should follow this template: + * + * "Mon Feb 12 11:35:33 UTC 2007" + */ +#define LIBCURL_TIMESTAMP "Wed Jan 27 07:32:44 UTC 2016" + +#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) +#define CURL_AT_LEAST_VERSION(x,y,z) \ + (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) + +#endif /* __CURL_CURLVER_H */ diff --git a/deps/libcurl/include/curl/easy.h b/deps/libcurl/include/curl/easy.h new file mode 100644 index 0000000000000000000000000000000000000000..c1e3e76096e3925821da279a2f2eae67ae61e0e8 --- /dev/null +++ b/deps/libcurl/include/curl/easy.h @@ -0,0 +1,102 @@ +#ifndef __CURL_EASY_H +#define __CURL_EASY_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2008, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN CURL *curl_easy_init(void); +CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); +CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); +CURL_EXTERN void curl_easy_cleanup(CURL *curl); + +/* + * NAME curl_easy_getinfo() + * + * DESCRIPTION + * + * Request internal information from the curl session with this function. The + * third argument MUST be a pointer to a long, a pointer to a char * or a + * pointer to a double (as the documentation describes elsewhere). The data + * pointed to will be filled in accordingly and can be relied upon only if the + * function returns CURLE_OK. This function is intended to get used *AFTER* a + * performed transfer, all results from this function are undefined until the + * transfer is completed. + */ +CURL_EXTERN CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...); + + +/* + * NAME curl_easy_duphandle() + * + * DESCRIPTION + * + * Creates a new curl session handle with the same options set for the handle + * passed in. Duplicating a handle could only be a matter of cloning data and + * options, internal state info and things like persistent connections cannot + * be transferred. It is useful in multithreaded applications when you can run + * curl_easy_duphandle() for each new thread to avoid a series of identical + * curl_easy_setopt() invokes in every thread. + */ +CURL_EXTERN CURL* curl_easy_duphandle(CURL *curl); + +/* + * NAME curl_easy_reset() + * + * DESCRIPTION + * + * Re-initializes a CURL handle to the default values. This puts back the + * handle to the same state as it was in when it was just created. + * + * It does keep: live connections, the Session ID cache, the DNS cache and the + * cookies. + */ +CURL_EXTERN void curl_easy_reset(CURL *curl); + +/* + * NAME curl_easy_recv() + * + * DESCRIPTION + * + * Receives data from the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen, + size_t *n); + +/* + * NAME curl_easy_send() + * + * DESCRIPTION + * + * Sends data over the connected socket. Use after successful + * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. + */ +CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer, + size_t buflen, size_t *n); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/deps/libcurl/include/curl/mprintf.h b/deps/libcurl/include/curl/mprintf.h new file mode 100644 index 0000000000000000000000000000000000000000..c6b0d7679a6cf970a7960af9f47a7485729598dc --- /dev/null +++ b/deps/libcurl/include/curl/mprintf.h @@ -0,0 +1,74 @@ +#ifndef __CURL_MPRINTF_H +#define __CURL_MPRINTF_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include +#include /* needed for FILE */ + +#include "curl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +CURL_EXTERN int curl_mprintf(const char *format, ...); +CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...); +CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...); +CURL_EXTERN int curl_msnprintf(char *buffer, size_t maxlength, + const char *format, ...); +CURL_EXTERN int curl_mvprintf(const char *format, va_list args); +CURL_EXTERN int curl_mvfprintf(FILE *fd, const char *format, va_list args); +CURL_EXTERN int curl_mvsprintf(char *buffer, const char *format, va_list args); +CURL_EXTERN int curl_mvsnprintf(char *buffer, size_t maxlength, + const char *format, va_list args); +CURL_EXTERN char *curl_maprintf(const char *format, ...); +CURL_EXTERN char *curl_mvaprintf(const char *format, va_list args); + +#ifdef _MPRINTF_REPLACE +# undef printf +# undef fprintf +# undef sprintf +# undef vsprintf +# undef snprintf +# undef vprintf +# undef vfprintf +# undef vsnprintf +# undef aprintf +# undef vaprintf +# define printf curl_mprintf +# define fprintf curl_mfprintf +# define sprintf curl_msprintf +# define vsprintf curl_mvsprintf +# define snprintf curl_msnprintf +# define vprintf curl_mvprintf +# define vfprintf curl_mvfprintf +# define vsnprintf curl_mvsnprintf +# define aprintf curl_maprintf +# define vaprintf curl_mvaprintf +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* __CURL_MPRINTF_H */ diff --git a/deps/libcurl/include/curl/multi.h b/deps/libcurl/include/curl/multi.h new file mode 100644 index 0000000000000000000000000000000000000000..36e2e940ecd440c0a14aadc0dc3eba1d5d53f186 --- /dev/null +++ b/deps/libcurl/include/curl/multi.h @@ -0,0 +1,435 @@ +#ifndef __CURL_MULTI_H +#define __CURL_MULTI_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ +/* + This is an "external" header file. Don't give away any internals here! + + GOALS + + o Enable a "pull" interface. The application that uses libcurl decides where + and when to ask libcurl to get/send data. + + o Enable multiple simultaneous transfers in the same thread without making it + complicated for the application. + + o Enable the application to select() on its own file descriptors and curl's + file descriptors simultaneous easily. + +*/ + +/* + * This header file should not really need to include "curl.h" since curl.h + * itself includes this file and we expect user applications to do #include + * without the need for especially including multi.h. + * + * For some reason we added this include here at one point, and rather than to + * break existing (wrongly written) libcurl applications, we leave it as-is + * but with this warning attached. + */ +#include "curl.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void CURLM; + +typedef enum { + CURLM_CALL_MULTI_PERFORM = -1, /* please call curl_multi_perform() or + curl_multi_socket*() soon */ + CURLM_OK, + CURLM_BAD_HANDLE, /* the passed-in handle is not a valid CURLM handle */ + CURLM_BAD_EASY_HANDLE, /* an easy handle was not good/valid */ + CURLM_OUT_OF_MEMORY, /* if you ever get this, you're in deep sh*t */ + CURLM_INTERNAL_ERROR, /* this is a libcurl bug */ + CURLM_BAD_SOCKET, /* the passed in socket argument did not match */ + CURLM_UNKNOWN_OPTION, /* curl_multi_setopt() with unsupported option */ + CURLM_ADDED_ALREADY, /* an easy handle already added to a multi handle was + attempted to get added - again */ + CURLM_LAST +} CURLMcode; + +/* just to make code nicer when using curl_multi_socket() you can now check + for CURLM_CALL_MULTI_SOCKET too in the same style it works for + curl_multi_perform() and CURLM_CALL_MULTI_PERFORM */ +#define CURLM_CALL_MULTI_SOCKET CURLM_CALL_MULTI_PERFORM + +/* bitmask bits for CURLMOPT_PIPELINING */ +#define CURLPIPE_NOTHING 0L +#define CURLPIPE_HTTP1 1L +#define CURLPIPE_MULTIPLEX 2L + +typedef enum { + CURLMSG_NONE, /* first, not used */ + CURLMSG_DONE, /* This easy handle has completed. 'result' contains + the CURLcode of the transfer */ + CURLMSG_LAST /* last, not used */ +} CURLMSG; + +struct CURLMsg { + CURLMSG msg; /* what this message means */ + CURL *easy_handle; /* the handle it concerns */ + union { + void *whatever; /* message-specific data */ + CURLcode result; /* return code for transfer */ + } data; +}; +typedef struct CURLMsg CURLMsg; + +/* Based on poll(2) structure and values. + * We don't use pollfd and POLL* constants explicitly + * to cover platforms without poll(). */ +#define CURL_WAIT_POLLIN 0x0001 +#define CURL_WAIT_POLLPRI 0x0002 +#define CURL_WAIT_POLLOUT 0x0004 + +struct curl_waitfd { + curl_socket_t fd; + short events; + short revents; /* not supported yet */ +}; + +/* + * Name: curl_multi_init() + * + * Desc: inititalize multi-style curl usage + * + * Returns: a new CURLM handle to use in all 'curl_multi' functions. + */ +CURL_EXTERN CURLM *curl_multi_init(void); + +/* + * Name: curl_multi_add_handle() + * + * Desc: add a standard curl handle to the multi stack + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_add_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_remove_handle() + * + * Desc: removes a curl handle from the multi stack again + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_remove_handle(CURLM *multi_handle, + CURL *curl_handle); + + /* + * Name: curl_multi_fdset() + * + * Desc: Ask curl for its fd_set sets. The app can use these to select() or + * poll() on. We want curl_multi_perform() called as soon as one of + * them are ready. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_fdset(CURLM *multi_handle, + fd_set *read_fd_set, + fd_set *write_fd_set, + fd_set *exc_fd_set, + int *max_fd); + +/* + * Name: curl_multi_wait() + * + * Desc: Poll on all fds within a CURLM set as well as any + * additional fds passed to the function. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_wait(CURLM *multi_handle, + struct curl_waitfd extra_fds[], + unsigned int extra_nfds, + int timeout_ms, + int *ret); + + /* + * Name: curl_multi_perform() + * + * Desc: When the app thinks there's data available for curl it calls this + * function to read/write whatever there is right now. This returns + * as soon as the reads and writes are done. This function does not + * require that there actually is data available for reading or that + * data can be written, it can be called just in case. It returns + * the number of handles that still transfer data in the second + * argument's integer-pointer. + * + * Returns: CURLMcode type, general multi error code. *NOTE* that this only + * returns errors etc regarding the whole multi stack. There might + * still have occurred problems on invidual transfers even when this + * returns OK. + */ +CURL_EXTERN CURLMcode curl_multi_perform(CURLM *multi_handle, + int *running_handles); + + /* + * Name: curl_multi_cleanup() + * + * Desc: Cleans up and removes a whole multi stack. It does not free or + * touch any individual easy handles in any way. We need to define + * in what state those handles will be if this function is called + * in the middle of a transfer. + * + * Returns: CURLMcode type, general multi error code. + */ +CURL_EXTERN CURLMcode curl_multi_cleanup(CURLM *multi_handle); + +/* + * Name: curl_multi_info_read() + * + * Desc: Ask the multi handle if there's any messages/informationals from + * the individual transfers. Messages include informationals such as + * error code from the transfer or just the fact that a transfer is + * completed. More details on these should be written down as well. + * + * Repeated calls to this function will return a new struct each + * time, until a special "end of msgs" struct is returned as a signal + * that there is no more to get at this point. + * + * The data the returned pointer points to will not survive calling + * curl_multi_cleanup(). + * + * The 'CURLMsg' struct is meant to be very simple and only contain + * very basic informations. If more involved information is wanted, + * we will provide the particular "transfer handle" in that struct + * and that should/could/would be used in subsequent + * curl_easy_getinfo() calls (or similar). The point being that we + * must never expose complex structs to applications, as then we'll + * undoubtably get backwards compatibility problems in the future. + * + * Returns: A pointer to a filled-in struct, or NULL if it failed or ran out + * of structs. It also writes the number of messages left in the + * queue (after this read) in the integer the second argument points + * to. + */ +CURL_EXTERN CURLMsg *curl_multi_info_read(CURLM *multi_handle, + int *msgs_in_queue); + +/* + * Name: curl_multi_strerror() + * + * Desc: The curl_multi_strerror function may be used to turn a CURLMcode + * value into the equivalent human readable error string. This is + * useful for printing meaningful error messages. + * + * Returns: A pointer to a zero-terminated error message. + */ +CURL_EXTERN const char *curl_multi_strerror(CURLMcode); + +/* + * Name: curl_multi_socket() and + * curl_multi_socket_all() + * + * Desc: An alternative version of curl_multi_perform() that allows the + * application to pass in one of the file descriptors that have been + * detected to have "action" on them and let libcurl perform. + * See man page for details. + */ +#define CURL_POLL_NONE 0 +#define CURL_POLL_IN 1 +#define CURL_POLL_OUT 2 +#define CURL_POLL_INOUT 3 +#define CURL_POLL_REMOVE 4 + +#define CURL_SOCKET_TIMEOUT CURL_SOCKET_BAD + +#define CURL_CSELECT_IN 0x01 +#define CURL_CSELECT_OUT 0x02 +#define CURL_CSELECT_ERR 0x04 + +typedef int (*curl_socket_callback)(CURL *easy, /* easy handle */ + curl_socket_t s, /* socket */ + int what, /* see above */ + void *userp, /* private callback + pointer */ + void *socketp); /* private socket + pointer */ +/* + * Name: curl_multi_timer_callback + * + * Desc: Called by libcurl whenever the library detects a change in the + * maximum number of milliseconds the app is allowed to wait before + * curl_multi_socket() or curl_multi_perform() must be called + * (to allow libcurl's timed events to take place). + * + * Returns: The callback should return zero. + */ +typedef int (*curl_multi_timer_callback)(CURLM *multi, /* multi handle */ + long timeout_ms, /* see above */ + void *userp); /* private callback + pointer */ + +CURL_EXTERN CURLMcode curl_multi_socket(CURLM *multi_handle, curl_socket_t s, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_action(CURLM *multi_handle, + curl_socket_t s, + int ev_bitmask, + int *running_handles); + +CURL_EXTERN CURLMcode curl_multi_socket_all(CURLM *multi_handle, + int *running_handles); + +#ifndef CURL_ALLOW_OLD_MULTI_SOCKET +/* This macro below was added in 7.16.3 to push users who recompile to use + the new curl_multi_socket_action() instead of the old curl_multi_socket() +*/ +#define curl_multi_socket(x,y,z) curl_multi_socket_action(x,y,0,z) +#endif + +/* + * Name: curl_multi_timeout() + * + * Desc: Returns the maximum number of milliseconds the app is allowed to + * wait before curl_multi_socket() or curl_multi_perform() must be + * called (to allow libcurl's timed events to take place). + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_timeout(CURLM *multi_handle, + long *milliseconds); + +#undef CINIT /* re-using the same name as in curl.h */ + +#ifdef CURL_ISOCPP +#define CINIT(name,type,num) CURLMOPT_ ## name = CURLOPTTYPE_ ## type + num +#else +/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ +#define LONG CURLOPTTYPE_LONG +#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT +#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT +#define OFF_T CURLOPTTYPE_OFF_T +#define CINIT(name,type,number) CURLMOPT_/**/name = type + number +#endif + +typedef enum { + /* This is the socket callback function pointer */ + CINIT(SOCKETFUNCTION, FUNCTIONPOINT, 1), + + /* This is the argument passed to the socket callback */ + CINIT(SOCKETDATA, OBJECTPOINT, 2), + + /* set to 1 to enable pipelining for this multi handle */ + CINIT(PIPELINING, LONG, 3), + + /* This is the timer callback function pointer */ + CINIT(TIMERFUNCTION, FUNCTIONPOINT, 4), + + /* This is the argument passed to the timer callback */ + CINIT(TIMERDATA, OBJECTPOINT, 5), + + /* maximum number of entries in the connection cache */ + CINIT(MAXCONNECTS, LONG, 6), + + /* maximum number of (pipelining) connections to one host */ + CINIT(MAX_HOST_CONNECTIONS, LONG, 7), + + /* maximum number of requests in a pipeline */ + CINIT(MAX_PIPELINE_LENGTH, LONG, 8), + + /* a connection with a content-length longer than this + will not be considered for pipelining */ + CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9), + + /* a connection with a chunk length longer than this + will not be considered for pipelining */ + CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10), + + /* a list of site names(+port) that are blacklisted from + pipelining */ + CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11), + + /* a list of server types that are blacklisted from + pipelining */ + CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12), + + /* maximum number of open connections in total */ + CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13), + + /* This is the server push callback function pointer */ + CINIT(PUSHFUNCTION, FUNCTIONPOINT, 14), + + /* This is the argument passed to the server push callback */ + CINIT(PUSHDATA, OBJECTPOINT, 15), + + CURLMOPT_LASTENTRY /* the last unused */ +} CURLMoption; + + +/* + * Name: curl_multi_setopt() + * + * Desc: Sets options for the multi handle. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_setopt(CURLM *multi_handle, + CURLMoption option, ...); + + +/* + * Name: curl_multi_assign() + * + * Desc: This function sets an association in the multi handle between the + * given socket and a private pointer of the application. This is + * (only) useful for curl_multi_socket uses. + * + * Returns: CURLM error code. + */ +CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle, + curl_socket_t sockfd, void *sockp); + + +/* + * Name: curl_push_callback + * + * Desc: This callback gets called when a new stream is being pushed by the + * server. It approves or denies the new stream. + * + * Returns: CURL_PUSH_OK or CURL_PUSH_DENY. + */ +#define CURL_PUSH_OK 0 +#define CURL_PUSH_DENY 1 + +struct curl_pushheaders; /* forward declaration only */ + +CURL_EXTERN char *curl_pushheader_bynum(struct curl_pushheaders *h, + size_t num); +CURL_EXTERN char *curl_pushheader_byname(struct curl_pushheaders *h, + const char *name); + +typedef int (*curl_push_callback)(CURL *parent, + CURL *easy, + size_t num_headers, + struct curl_pushheaders *headers, + void *userp); + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif diff --git a/deps/libcurl/include/curl/stdcheaders.h b/deps/libcurl/include/curl/stdcheaders.h new file mode 100644 index 0000000000000000000000000000000000000000..ad82ef6335d6167aecf8b9e49d68c462ea028bfd --- /dev/null +++ b/deps/libcurl/include/curl/stdcheaders.h @@ -0,0 +1,33 @@ +#ifndef __STDC_HEADERS_H +#define __STDC_HEADERS_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2010, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include + +size_t fread (void *, size_t, size_t, FILE *); +size_t fwrite (const void *, size_t, size_t, FILE *); + +int strcasecmp(const char *, const char *); +int strncasecmp(const char *, const char *, size_t); + +#endif /* __STDC_HEADERS_H */ diff --git a/deps/libcurl/include/curl/typecheck-gcc.h b/deps/libcurl/include/curl/typecheck-gcc.h new file mode 100644 index 0000000000000000000000000000000000000000..2e24db0ff59c40b137ec89a9d8a3f571ed4791b2 --- /dev/null +++ b/deps/libcurl/include/curl/typecheck-gcc.h @@ -0,0 +1,622 @@ +#ifndef __CURL_TYPECHECK_GCC_H +#define __CURL_TYPECHECK_GCC_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +/* wraps curl_easy_setopt() with typechecking */ + +/* To add a new kind of warning, add an + * if(_curl_is_sometype_option(_curl_opt)) + * if(!_curl_is_sometype(value)) + * _curl_easy_setopt_err_sometype(); + * block and define _curl_is_sometype_option, _curl_is_sometype and + * _curl_easy_setopt_err_sometype below + * + * NOTE: We use two nested 'if' statements here instead of the && operator, in + * order to work around gcc bug #32061. It affects only gcc 4.3.x/4.4.x + * when compiling with -Wlogical-op. + * + * To add an option that uses the same type as an existing option, you'll just + * need to extend the appropriate _curl_*_option macro + */ +#define curl_easy_setopt(handle, option, value) \ +__extension__ ({ \ + __typeof__ (option) _curl_opt = option; \ + if(__builtin_constant_p(_curl_opt)) { \ + if(_curl_is_long_option(_curl_opt)) \ + if(!_curl_is_long(value)) \ + _curl_easy_setopt_err_long(); \ + if(_curl_is_off_t_option(_curl_opt)) \ + if(!_curl_is_off_t(value)) \ + _curl_easy_setopt_err_curl_off_t(); \ + if(_curl_is_string_option(_curl_opt)) \ + if(!_curl_is_string(value)) \ + _curl_easy_setopt_err_string(); \ + if(_curl_is_write_cb_option(_curl_opt)) \ + if(!_curl_is_write_cb(value)) \ + _curl_easy_setopt_err_write_callback(); \ + if((_curl_opt) == CURLOPT_READFUNCTION) \ + if(!_curl_is_read_cb(value)) \ + _curl_easy_setopt_err_read_cb(); \ + if((_curl_opt) == CURLOPT_IOCTLFUNCTION) \ + if(!_curl_is_ioctl_cb(value)) \ + _curl_easy_setopt_err_ioctl_cb(); \ + if((_curl_opt) == CURLOPT_SOCKOPTFUNCTION) \ + if(!_curl_is_sockopt_cb(value)) \ + _curl_easy_setopt_err_sockopt_cb(); \ + if((_curl_opt) == CURLOPT_OPENSOCKETFUNCTION) \ + if(!_curl_is_opensocket_cb(value)) \ + _curl_easy_setopt_err_opensocket_cb(); \ + if((_curl_opt) == CURLOPT_PROGRESSFUNCTION) \ + if(!_curl_is_progress_cb(value)) \ + _curl_easy_setopt_err_progress_cb(); \ + if((_curl_opt) == CURLOPT_DEBUGFUNCTION) \ + if(!_curl_is_debug_cb(value)) \ + _curl_easy_setopt_err_debug_cb(); \ + if((_curl_opt) == CURLOPT_SSL_CTX_FUNCTION) \ + if(!_curl_is_ssl_ctx_cb(value)) \ + _curl_easy_setopt_err_ssl_ctx_cb(); \ + if(_curl_is_conv_cb_option(_curl_opt)) \ + if(!_curl_is_conv_cb(value)) \ + _curl_easy_setopt_err_conv_cb(); \ + if((_curl_opt) == CURLOPT_SEEKFUNCTION) \ + if(!_curl_is_seek_cb(value)) \ + _curl_easy_setopt_err_seek_cb(); \ + if(_curl_is_cb_data_option(_curl_opt)) \ + if(!_curl_is_cb_data(value)) \ + _curl_easy_setopt_err_cb_data(); \ + if((_curl_opt) == CURLOPT_ERRORBUFFER) \ + if(!_curl_is_error_buffer(value)) \ + _curl_easy_setopt_err_error_buffer(); \ + if((_curl_opt) == CURLOPT_STDERR) \ + if(!_curl_is_FILE(value)) \ + _curl_easy_setopt_err_FILE(); \ + if(_curl_is_postfields_option(_curl_opt)) \ + if(!_curl_is_postfields(value)) \ + _curl_easy_setopt_err_postfields(); \ + if((_curl_opt) == CURLOPT_HTTPPOST) \ + if(!_curl_is_arr((value), struct curl_httppost)) \ + _curl_easy_setopt_err_curl_httpost(); \ + if(_curl_is_slist_option(_curl_opt)) \ + if(!_curl_is_arr((value), struct curl_slist)) \ + _curl_easy_setopt_err_curl_slist(); \ + if((_curl_opt) == CURLOPT_SHARE) \ + if(!_curl_is_ptr((value), CURLSH)) \ + _curl_easy_setopt_err_CURLSH(); \ + } \ + curl_easy_setopt(handle, _curl_opt, value); \ +}) + +/* wraps curl_easy_getinfo() with typechecking */ +/* FIXME: don't allow const pointers */ +#define curl_easy_getinfo(handle, info, arg) \ +__extension__ ({ \ + __typeof__ (info) _curl_info = info; \ + if(__builtin_constant_p(_curl_info)) { \ + if(_curl_is_string_info(_curl_info)) \ + if(!_curl_is_arr((arg), char *)) \ + _curl_easy_getinfo_err_string(); \ + if(_curl_is_long_info(_curl_info)) \ + if(!_curl_is_arr((arg), long)) \ + _curl_easy_getinfo_err_long(); \ + if(_curl_is_double_info(_curl_info)) \ + if(!_curl_is_arr((arg), double)) \ + _curl_easy_getinfo_err_double(); \ + if(_curl_is_slist_info(_curl_info)) \ + if(!_curl_is_arr((arg), struct curl_slist *)) \ + _curl_easy_getinfo_err_curl_slist(); \ + } \ + curl_easy_getinfo(handle, _curl_info, arg); \ +}) + +/* TODO: typechecking for curl_share_setopt() and curl_multi_setopt(), + * for now just make sure that the functions are called with three + * arguments + */ +#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) +#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) + + +/* the actual warnings, triggered by calling the _curl_easy_setopt_err* + * functions */ + +/* To define a new warning, use _CURL_WARNING(identifier, "message") */ +#define _CURL_WARNING(id, message) \ + static void __attribute__((__warning__(message))) \ + __attribute__((__unused__)) __attribute__((__noinline__)) \ + id(void) { __asm__(""); } + +_CURL_WARNING(_curl_easy_setopt_err_long, + "curl_easy_setopt expects a long argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_off_t, + "curl_easy_setopt expects a curl_off_t argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_string, + "curl_easy_setopt expects a " + "string (char* or char[]) argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_write_callback, + "curl_easy_setopt expects a curl_write_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_read_cb, + "curl_easy_setopt expects a curl_read_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ioctl_cb, + "curl_easy_setopt expects a curl_ioctl_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_sockopt_cb, + "curl_easy_setopt expects a curl_sockopt_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_opensocket_cb, + "curl_easy_setopt expects a " + "curl_opensocket_callback argument for this option" + ) +_CURL_WARNING(_curl_easy_setopt_err_progress_cb, + "curl_easy_setopt expects a curl_progress_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_debug_cb, + "curl_easy_setopt expects a curl_debug_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_ssl_ctx_cb, + "curl_easy_setopt expects a curl_ssl_ctx_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_conv_cb, + "curl_easy_setopt expects a curl_conv_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_seek_cb, + "curl_easy_setopt expects a curl_seek_callback argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_cb_data, + "curl_easy_setopt expects a " + "private data pointer as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_error_buffer, + "curl_easy_setopt expects a " + "char buffer of CURL_ERROR_SIZE as argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_FILE, + "curl_easy_setopt expects a FILE* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_postfields, + "curl_easy_setopt expects a void* or char* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_httpost, + "curl_easy_setopt expects a struct curl_httppost* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_curl_slist, + "curl_easy_setopt expects a struct curl_slist* argument for this option") +_CURL_WARNING(_curl_easy_setopt_err_CURLSH, + "curl_easy_setopt expects a CURLSH* argument for this option") + +_CURL_WARNING(_curl_easy_getinfo_err_string, + "curl_easy_getinfo expects a pointer to char * for this info") +_CURL_WARNING(_curl_easy_getinfo_err_long, + "curl_easy_getinfo expects a pointer to long for this info") +_CURL_WARNING(_curl_easy_getinfo_err_double, + "curl_easy_getinfo expects a pointer to double for this info") +_CURL_WARNING(_curl_easy_getinfo_err_curl_slist, + "curl_easy_getinfo expects a pointer to struct curl_slist * for this info") + +/* groups of curl_easy_setops options that take the same type of argument */ + +/* To add a new option to one of the groups, just add + * (option) == CURLOPT_SOMETHING + * to the or-expression. If the option takes a long or curl_off_t, you don't + * have to do anything + */ + +/* evaluates to true if option takes a long argument */ +#define _curl_is_long_option(option) \ + (0 < (option) && (option) < CURLOPTTYPE_OBJECTPOINT) + +#define _curl_is_off_t_option(option) \ + ((option) > CURLOPTTYPE_OFF_T) + +/* evaluates to true if option takes a char* argument */ +#define _curl_is_string_option(option) \ + ((option) == CURLOPT_ACCEPT_ENCODING || \ + (option) == CURLOPT_CAINFO || \ + (option) == CURLOPT_CAPATH || \ + (option) == CURLOPT_COOKIE || \ + (option) == CURLOPT_COOKIEFILE || \ + (option) == CURLOPT_COOKIEJAR || \ + (option) == CURLOPT_COOKIELIST || \ + (option) == CURLOPT_CRLFILE || \ + (option) == CURLOPT_CUSTOMREQUEST || \ + (option) == CURLOPT_DEFAULT_PROTOCOL || \ + (option) == CURLOPT_DNS_INTERFACE || \ + (option) == CURLOPT_DNS_LOCAL_IP4 || \ + (option) == CURLOPT_DNS_LOCAL_IP6 || \ + (option) == CURLOPT_DNS_SERVERS || \ + (option) == CURLOPT_EGDSOCKET || \ + (option) == CURLOPT_FTPPORT || \ + (option) == CURLOPT_FTP_ACCOUNT || \ + (option) == CURLOPT_FTP_ALTERNATIVE_TO_USER || \ + (option) == CURLOPT_INTERFACE || \ + (option) == CURLOPT_ISSUERCERT || \ + (option) == CURLOPT_KEYPASSWD || \ + (option) == CURLOPT_KRBLEVEL || \ + (option) == CURLOPT_LOGIN_OPTIONS || \ + (option) == CURLOPT_MAIL_AUTH || \ + (option) == CURLOPT_MAIL_FROM || \ + (option) == CURLOPT_NETRC_FILE || \ + (option) == CURLOPT_NOPROXY || \ + (option) == CURLOPT_PASSWORD || \ + (option) == CURLOPT_PINNEDPUBLICKEY || \ + (option) == CURLOPT_PROXY || \ + (option) == CURLOPT_PROXYPASSWORD || \ + (option) == CURLOPT_PROXYUSERNAME || \ + (option) == CURLOPT_PROXYUSERPWD || \ + (option) == CURLOPT_PROXY_SERVICE_NAME || \ + (option) == CURLOPT_RANDOM_FILE || \ + (option) == CURLOPT_RANGE || \ + (option) == CURLOPT_REFERER || \ + (option) == CURLOPT_RTSP_SESSION_ID || \ + (option) == CURLOPT_RTSP_STREAM_URI || \ + (option) == CURLOPT_RTSP_TRANSPORT || \ + (option) == CURLOPT_SERVICE_NAME || \ + (option) == CURLOPT_SOCKS5_GSSAPI_SERVICE || \ + (option) == CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 || \ + (option) == CURLOPT_SSH_KNOWNHOSTS || \ + (option) == CURLOPT_SSH_PRIVATE_KEYFILE || \ + (option) == CURLOPT_SSH_PUBLIC_KEYFILE || \ + (option) == CURLOPT_SSLCERT || \ + (option) == CURLOPT_SSLCERTTYPE || \ + (option) == CURLOPT_SSLENGINE || \ + (option) == CURLOPT_SSLKEY || \ + (option) == CURLOPT_SSLKEYTYPE || \ + (option) == CURLOPT_SSL_CIPHER_LIST || \ + (option) == CURLOPT_TLSAUTH_PASSWORD || \ + (option) == CURLOPT_TLSAUTH_TYPE || \ + (option) == CURLOPT_TLSAUTH_USERNAME || \ + (option) == CURLOPT_UNIX_SOCKET_PATH || \ + (option) == CURLOPT_URL || \ + (option) == CURLOPT_USERAGENT || \ + (option) == CURLOPT_USERNAME || \ + (option) == CURLOPT_USERPWD || \ + (option) == CURLOPT_XOAUTH2_BEARER || \ + 0) + +/* evaluates to true if option takes a curl_write_callback argument */ +#define _curl_is_write_cb_option(option) \ + ((option) == CURLOPT_HEADERFUNCTION || \ + (option) == CURLOPT_WRITEFUNCTION) + +/* evaluates to true if option takes a curl_conv_callback argument */ +#define _curl_is_conv_cb_option(option) \ + ((option) == CURLOPT_CONV_TO_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_NETWORK_FUNCTION || \ + (option) == CURLOPT_CONV_FROM_UTF8_FUNCTION) + +/* evaluates to true if option takes a data argument to pass to a callback */ +#define _curl_is_cb_data_option(option) \ + ((option) == CURLOPT_CHUNK_DATA || \ + (option) == CURLOPT_CLOSESOCKETDATA || \ + (option) == CURLOPT_DEBUGDATA || \ + (option) == CURLOPT_FNMATCH_DATA || \ + (option) == CURLOPT_HEADERDATA || \ + (option) == CURLOPT_INTERLEAVEDATA || \ + (option) == CURLOPT_IOCTLDATA || \ + (option) == CURLOPT_OPENSOCKETDATA || \ + (option) == CURLOPT_PRIVATE || \ + (option) == CURLOPT_PROGRESSDATA || \ + (option) == CURLOPT_READDATA || \ + (option) == CURLOPT_SEEKDATA || \ + (option) == CURLOPT_SOCKOPTDATA || \ + (option) == CURLOPT_SSH_KEYDATA || \ + (option) == CURLOPT_SSL_CTX_DATA || \ + (option) == CURLOPT_WRITEDATA || \ + 0) + +/* evaluates to true if option takes a POST data argument (void* or char*) */ +#define _curl_is_postfields_option(option) \ + ((option) == CURLOPT_POSTFIELDS || \ + (option) == CURLOPT_COPYPOSTFIELDS || \ + 0) + +/* evaluates to true if option takes a struct curl_slist * argument */ +#define _curl_is_slist_option(option) \ + ((option) == CURLOPT_HTTP200ALIASES || \ + (option) == CURLOPT_HTTPHEADER || \ + (option) == CURLOPT_MAIL_RCPT || \ + (option) == CURLOPT_POSTQUOTE || \ + (option) == CURLOPT_PREQUOTE || \ + (option) == CURLOPT_PROXYHEADER || \ + (option) == CURLOPT_QUOTE || \ + (option) == CURLOPT_RESOLVE || \ + (option) == CURLOPT_TELNETOPTIONS || \ + 0) + +/* groups of curl_easy_getinfo infos that take the same type of argument */ + +/* evaluates to true if info expects a pointer to char * argument */ +#define _curl_is_string_info(info) \ + (CURLINFO_STRING < (info) && (info) < CURLINFO_LONG) + +/* evaluates to true if info expects a pointer to long argument */ +#define _curl_is_long_info(info) \ + (CURLINFO_LONG < (info) && (info) < CURLINFO_DOUBLE) + +/* evaluates to true if info expects a pointer to double argument */ +#define _curl_is_double_info(info) \ + (CURLINFO_DOUBLE < (info) && (info) < CURLINFO_SLIST) + +/* true if info expects a pointer to struct curl_slist * argument */ +#define _curl_is_slist_info(info) \ + (CURLINFO_SLIST < (info)) + + +/* typecheck helpers -- check whether given expression has requested type*/ + +/* For pointers, you can use the _curl_is_ptr/_curl_is_arr macros, + * otherwise define a new macro. Search for __builtin_types_compatible_p + * in the GCC manual. + * NOTE: these macros MUST NOT EVALUATE their arguments! The argument is + * the actual expression passed to the curl_easy_setopt macro. This + * means that you can only apply the sizeof and __typeof__ operators, no + * == or whatsoever. + */ + +/* XXX: should evaluate to true iff expr is a pointer */ +#define _curl_is_any_ptr(expr) \ + (sizeof(expr) == sizeof(void*)) + +/* evaluates to true if expr is NULL */ +/* XXX: must not evaluate expr, so this check is not accurate */ +#define _curl_is_NULL(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), __typeof__(NULL))) + +/* evaluates to true if expr is type*, const type* or NULL */ +#define _curl_is_ptr(expr, type) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), type *) || \ + __builtin_types_compatible_p(__typeof__(expr), const type *)) + +/* evaluates to true if expr is one of type[], type*, NULL or const type* */ +#define _curl_is_arr(expr, type) \ + (_curl_is_ptr((expr), type) || \ + __builtin_types_compatible_p(__typeof__(expr), type [])) + +/* evaluates to true if expr is a string */ +#define _curl_is_string(expr) \ + (_curl_is_arr((expr), char) || \ + _curl_is_arr((expr), signed char) || \ + _curl_is_arr((expr), unsigned char)) + +/* evaluates to true if expr is a long (no matter the signedness) + * XXX: for now, int is also accepted (and therefore short and char, which + * are promoted to int when passed to a variadic function) */ +#define _curl_is_long(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), long) || \ + __builtin_types_compatible_p(__typeof__(expr), signed long) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned long) || \ + __builtin_types_compatible_p(__typeof__(expr), int) || \ + __builtin_types_compatible_p(__typeof__(expr), signed int) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned int) || \ + __builtin_types_compatible_p(__typeof__(expr), short) || \ + __builtin_types_compatible_p(__typeof__(expr), signed short) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned short) || \ + __builtin_types_compatible_p(__typeof__(expr), char) || \ + __builtin_types_compatible_p(__typeof__(expr), signed char) || \ + __builtin_types_compatible_p(__typeof__(expr), unsigned char)) + +/* evaluates to true if expr is of type curl_off_t */ +#define _curl_is_off_t(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), curl_off_t)) + +/* evaluates to true if expr is abuffer suitable for CURLOPT_ERRORBUFFER */ +/* XXX: also check size of an char[] array? */ +#define _curl_is_error_buffer(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), char *) || \ + __builtin_types_compatible_p(__typeof__(expr), char[])) + +/* evaluates to true if expr is of type (const) void* or (const) FILE* */ +#if 0 +#define _curl_is_cb_data(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_ptr((expr), FILE)) +#else /* be less strict */ +#define _curl_is_cb_data(expr) \ + _curl_is_any_ptr(expr) +#endif + +/* evaluates to true if expr is of type FILE* */ +#define _curl_is_FILE(expr) \ + (__builtin_types_compatible_p(__typeof__(expr), FILE *)) + +/* evaluates to true if expr can be passed as POST data (void* or char*) */ +#define _curl_is_postfields(expr) \ + (_curl_is_ptr((expr), void) || \ + _curl_is_arr((expr), char)) + +/* FIXME: the whole callback checking is messy... + * The idea is to tolerate char vs. void and const vs. not const + * pointers in arguments at least + */ +/* helper: __builtin_types_compatible_p distinguishes between functions and + * function pointers, hide it */ +#define _curl_callback_compatible(func, type) \ + (__builtin_types_compatible_p(__typeof__(func), type) || \ + __builtin_types_compatible_p(__typeof__(func), type*)) + +/* evaluates to true if expr is of type curl_read_callback or "similar" */ +#define _curl_is_read_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), __typeof__(fread)) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_read_callback) || \ + _curl_callback_compatible((expr), _curl_read_callback1) || \ + _curl_callback_compatible((expr), _curl_read_callback2) || \ + _curl_callback_compatible((expr), _curl_read_callback3) || \ + _curl_callback_compatible((expr), _curl_read_callback4) || \ + _curl_callback_compatible((expr), _curl_read_callback5) || \ + _curl_callback_compatible((expr), _curl_read_callback6)) +typedef size_t (_curl_read_callback1)(char *, size_t, size_t, void*); +typedef size_t (_curl_read_callback2)(char *, size_t, size_t, const void*); +typedef size_t (_curl_read_callback3)(char *, size_t, size_t, FILE*); +typedef size_t (_curl_read_callback4)(void *, size_t, size_t, void*); +typedef size_t (_curl_read_callback5)(void *, size_t, size_t, const void*); +typedef size_t (_curl_read_callback6)(void *, size_t, size_t, FILE*); + +/* evaluates to true if expr is of type curl_write_callback or "similar" */ +#define _curl_is_write_cb(expr) \ + (_curl_is_read_cb(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), __typeof__(fwrite)) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_write_callback) || \ + _curl_callback_compatible((expr), _curl_write_callback1) || \ + _curl_callback_compatible((expr), _curl_write_callback2) || \ + _curl_callback_compatible((expr), _curl_write_callback3) || \ + _curl_callback_compatible((expr), _curl_write_callback4) || \ + _curl_callback_compatible((expr), _curl_write_callback5) || \ + _curl_callback_compatible((expr), _curl_write_callback6)) +typedef size_t (_curl_write_callback1)(const char *, size_t, size_t, void*); +typedef size_t (_curl_write_callback2)(const char *, size_t, size_t, + const void*); +typedef size_t (_curl_write_callback3)(const char *, size_t, size_t, FILE*); +typedef size_t (_curl_write_callback4)(const void *, size_t, size_t, void*); +typedef size_t (_curl_write_callback5)(const void *, size_t, size_t, + const void*); +typedef size_t (_curl_write_callback6)(const void *, size_t, size_t, FILE*); + +/* evaluates to true if expr is of type curl_ioctl_callback or "similar" */ +#define _curl_is_ioctl_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_ioctl_callback) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback1) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback2) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback3) || \ + _curl_callback_compatible((expr), _curl_ioctl_callback4)) +typedef curlioerr (_curl_ioctl_callback1)(CURL *, int, void*); +typedef curlioerr (_curl_ioctl_callback2)(CURL *, int, const void*); +typedef curlioerr (_curl_ioctl_callback3)(CURL *, curliocmd, void*); +typedef curlioerr (_curl_ioctl_callback4)(CURL *, curliocmd, const void*); + +/* evaluates to true if expr is of type curl_sockopt_callback or "similar" */ +#define _curl_is_sockopt_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_sockopt_callback) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback1) || \ + _curl_callback_compatible((expr), _curl_sockopt_callback2)) +typedef int (_curl_sockopt_callback1)(void *, curl_socket_t, curlsocktype); +typedef int (_curl_sockopt_callback2)(const void *, curl_socket_t, + curlsocktype); + +/* evaluates to true if expr is of type curl_opensocket_callback or + "similar" */ +#define _curl_is_opensocket_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_opensocket_callback) ||\ + _curl_callback_compatible((expr), _curl_opensocket_callback1) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback2) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback3) || \ + _curl_callback_compatible((expr), _curl_opensocket_callback4)) +typedef curl_socket_t (_curl_opensocket_callback1) + (void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (_curl_opensocket_callback2) + (void *, curlsocktype, const struct curl_sockaddr *); +typedef curl_socket_t (_curl_opensocket_callback3) + (const void *, curlsocktype, struct curl_sockaddr *); +typedef curl_socket_t (_curl_opensocket_callback4) + (const void *, curlsocktype, const struct curl_sockaddr *); + +/* evaluates to true if expr is of type curl_progress_callback or "similar" */ +#define _curl_is_progress_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_progress_callback) || \ + _curl_callback_compatible((expr), _curl_progress_callback1) || \ + _curl_callback_compatible((expr), _curl_progress_callback2)) +typedef int (_curl_progress_callback1)(void *, + double, double, double, double); +typedef int (_curl_progress_callback2)(const void *, + double, double, double, double); + +/* evaluates to true if expr is of type curl_debug_callback or "similar" */ +#define _curl_is_debug_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_debug_callback) || \ + _curl_callback_compatible((expr), _curl_debug_callback1) || \ + _curl_callback_compatible((expr), _curl_debug_callback2) || \ + _curl_callback_compatible((expr), _curl_debug_callback3) || \ + _curl_callback_compatible((expr), _curl_debug_callback4) || \ + _curl_callback_compatible((expr), _curl_debug_callback5) || \ + _curl_callback_compatible((expr), _curl_debug_callback6) || \ + _curl_callback_compatible((expr), _curl_debug_callback7) || \ + _curl_callback_compatible((expr), _curl_debug_callback8)) +typedef int (_curl_debug_callback1) (CURL *, + curl_infotype, char *, size_t, void *); +typedef int (_curl_debug_callback2) (CURL *, + curl_infotype, char *, size_t, const void *); +typedef int (_curl_debug_callback3) (CURL *, + curl_infotype, const char *, size_t, void *); +typedef int (_curl_debug_callback4) (CURL *, + curl_infotype, const char *, size_t, const void *); +typedef int (_curl_debug_callback5) (CURL *, + curl_infotype, unsigned char *, size_t, void *); +typedef int (_curl_debug_callback6) (CURL *, + curl_infotype, unsigned char *, size_t, const void *); +typedef int (_curl_debug_callback7) (CURL *, + curl_infotype, const unsigned char *, size_t, void *); +typedef int (_curl_debug_callback8) (CURL *, + curl_infotype, const unsigned char *, size_t, const void *); + +/* evaluates to true if expr is of type curl_ssl_ctx_callback or "similar" */ +/* this is getting even messier... */ +#define _curl_is_ssl_ctx_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_ssl_ctx_callback) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback1) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback2) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback3) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback4) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback5) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback6) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback7) || \ + _curl_callback_compatible((expr), _curl_ssl_ctx_callback8)) +typedef CURLcode (_curl_ssl_ctx_callback1)(CURL *, void *, void *); +typedef CURLcode (_curl_ssl_ctx_callback2)(CURL *, void *, const void *); +typedef CURLcode (_curl_ssl_ctx_callback3)(CURL *, const void *, void *); +typedef CURLcode (_curl_ssl_ctx_callback4)(CURL *, const void *, const void *); +#ifdef HEADER_SSL_H +/* hack: if we included OpenSSL's ssl.h, we know about SSL_CTX + * this will of course break if we're included before OpenSSL headers... + */ +typedef CURLcode (_curl_ssl_ctx_callback5)(CURL *, SSL_CTX, void *); +typedef CURLcode (_curl_ssl_ctx_callback6)(CURL *, SSL_CTX, const void *); +typedef CURLcode (_curl_ssl_ctx_callback7)(CURL *, const SSL_CTX, void *); +typedef CURLcode (_curl_ssl_ctx_callback8)(CURL *, const SSL_CTX, + const void *); +#else +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback5; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback6; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback7; +typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback8; +#endif + +/* evaluates to true if expr is of type curl_conv_callback or "similar" */ +#define _curl_is_conv_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_conv_callback) || \ + _curl_callback_compatible((expr), _curl_conv_callback1) || \ + _curl_callback_compatible((expr), _curl_conv_callback2) || \ + _curl_callback_compatible((expr), _curl_conv_callback3) || \ + _curl_callback_compatible((expr), _curl_conv_callback4)) +typedef CURLcode (*_curl_conv_callback1)(char *, size_t length); +typedef CURLcode (*_curl_conv_callback2)(const char *, size_t length); +typedef CURLcode (*_curl_conv_callback3)(void *, size_t length); +typedef CURLcode (*_curl_conv_callback4)(const void *, size_t length); + +/* evaluates to true if expr is of type curl_seek_callback or "similar" */ +#define _curl_is_seek_cb(expr) \ + (_curl_is_NULL(expr) || \ + __builtin_types_compatible_p(__typeof__(expr), curl_seek_callback) || \ + _curl_callback_compatible((expr), _curl_seek_callback1) || \ + _curl_callback_compatible((expr), _curl_seek_callback2)) +typedef CURLcode (*_curl_seek_callback1)(void *, curl_off_t, int); +typedef CURLcode (*_curl_seek_callback2)(const void *, curl_off_t, int); + + +#endif /* __CURL_TYPECHECK_GCC_H */ diff --git a/deps/libcurl/lib/libcurl.a b/deps/libcurl/lib/libcurl.a new file mode 100644 index 0000000000000000000000000000000000000000..c63ac70946dddefb5b3b5c55cf82d693b731cad6 Binary files /dev/null and b/deps/libcurl/lib/libcurl.a differ diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md index 766a428f1bfa8559dfb86701195868a8b6821ebc..20f9bcb7307c68f1330519d8f6ed249419bf46db 100644 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md @@ -128,5 +128,18 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 ## [培训和FAQ](https://www.taosdata.com/cn/faq) -- [FAQ](https://www.taosdata.com/cn/documentation20/faq)：常见问题与答案 -- [应用案列](https://www.taosdata.com/cn/blog/?categories=4)：一些使用实例来解释如何使用TDengine + + diff --git a/documentation20/webdocs/markdowndocs/Getting Started-ch.md b/documentation20/webdocs/markdowndocs/Getting Started-ch.md index 9f9d3a2ec292af8745cde58f7467a63ab4de33fe..b53c014ba64fb203cb8e6943297fe6117034385c 100644 --- a/documentation20/webdocs/markdowndocs/Getting Started-ch.md +++ b/documentation20/webdocs/markdowndocs/Getting Started-ch.md @@ -20,7 +20,7 @@ TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。 - TDengine-server-2.0.10.0-Linux-x64.deb (2.7M) - TDengine-server-2.0.10.0-Linux-x64.tar.gz (4.5M) -具体的安装过程，请参见TDengine多种安装包的安装和卸载。 +具体的安装过程，请参见TDengine多种安装包的安装和卸载以及视频教程。 ## 轻松启动 diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index 27105bdb901842d1960528de0050d84a66923fb5..ea1be899a85fe6bb31ab03674ab496d7b301432f 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -4,6 +4,8 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个具体的应用场景，需要考虑库的设计，超级表和普通表的设计。本节不讨论细致的语法规则，只介绍概念。 +关于数据建模请参考视频教程。 + ## 创建库 不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： @@ -59,3 +61,4 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。 TDengine建议尽可能采用多列模型，因为插入效率以及存储效率更高。但对于有些场景，一个采集点的采集量的种类经常变化，这个时候，如果采用多列模型，就需要频繁修改超级表的结构定义，让应用变的复杂，这个时候，采用单列模型会显得简单。 + diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 60ac6e4c2edb67fffcc2a729b452e75323505fab..89f6a64f192c65ae422c6fc52600040d439456a7 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -6,6 +6,8 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 +关于集群搭建请参考视频教程。 + ## 准备工作 **第零步**：规划集群所有物理节点的FQDN，将规划好的FQDN分别添加到每个物理节点的/etc/hostname；修改每个物理节点的/etc/hosts，将所有集群物理节点的IP与FQDN的对应添加好。【如部署了DNS，请联系网络管理员在DNS上做好相关配置】 @@ -226,3 +228,4 @@ SHOW MNODES; 如果副本数为偶数，当一个vnode group里一半vnode不工作时，是无法从中选出master的。同理，一半mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 TDengine提供一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/)，在TDengine Arbitrator Linux一节中，选择适合的版本下载并安装。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。如果副本数为奇数，即使配置了arbitrator, 系统也不会去建立连接。 + diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md index 821b4c23bff5409995c216e05cc3bde41ca6a717..cc6287953ad0561841d7435148ecae07d28b6ca8 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-ch.md @@ -187,7 +187,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine - pass：密码 - db：数据库名字，如果用户没有提供，也可以正常连接，用户可以通过该连接创建新的数据库，如果用户提供了数据库名字，则说明该数据库用户已经创建好，缺省使用该数据库 - port：端口号 - + 返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。 - `char *taos_get_server_info(TAOS *taos)` @@ -336,7 +336,7 @@ TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线 - `TAOS_RES* taos_stmt_use_result(TAOS_STMT *stmt)` 获取语句的结果集。结果集的使用方式与非参数化调用时一致，使用完成后，应对此结果集调用 `taos_free_result`以释放资源。 - + - `int taos_stmt_close(TAOS_STMT *stmt)` 执行完毕，释放所有资源。 @@ -354,7 +354,7 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * stime：是流式计算开始的时间，如果是0，表示从现在开始，如果不为零，表示从指定的时间开始计算（UTC时间从1970/1/1算起的毫秒数） * param：是应用提供的用于回调的一个参数，回调时，提供给应用 * callback: 第二个回调函数，会在连续查询自动停止时被调用。 - + 返回值为NULL，表示创建成功，返回值不为空，表示成功。 - `void taos_close_stream (TAOS_STREAM *tstr)` @@ -394,6 +394,8 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 ## Python Connector +Python连接器的使用参见视频教程 + ### 安装准备 * 应用驱动安装请参考安装连接器驱动步骤。 * 已安装python 2.7 or >= 3.4 @@ -433,7 +435,7 @@ python -m pip install python3\ * 导入TDengine客户端模块 ```python -import taos +import taos ``` * 获取连接并获取游标对象 ```python @@ -445,7 +447,7 @@ c1 = conn.cursor() * 写入数据 ```python import datetime - + # 创建数据库 c1.execute('create database db') c1.execute('use db') @@ -473,7 +475,7 @@ numOfRows = c1.rowcount numOfCols = len(c1.description) for irow in range(numOfRows): print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])) - + # 直接使用cursor 循环拉取查询结果 c1.execute('select * from tb') for data in c1: @@ -534,7 +536,7 @@ conn.close() ## RESTful Connector -为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。 +为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。RESTful连接器的使用参见视频教程。 ### HTTP请求格式 @@ -779,7 +781,7 @@ https://www.taosdata.com/blog/2020/11/02/1901.html TDengine提供了GO驱动程序`taosSql`。 `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`。 -使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go。 +使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go 以及视频教程。 ```Go import ( @@ -796,7 +798,7 @@ go env -w GOPROXY=https://goproxy.io,direct ### 常用API -- sql.Open(DRIVER_NAME string, dataSourceName string) *DB` +- `sql.Open(DRIVER_NAME string, dataSourceName string) *DB` 该API用来打开DB，返回一个类型为*DB的对象，一般情况下，DRIVER_NAME设置为字符串`taosSql`, dataSourceName设置为字符串`user:password@/tcp(host:port)/dbname`，如果客户想要用多个goroutine并发访问TDengine, 那么需要在各个goroutine中分别创建一个sql.Open对象并用之访问TDengine @@ -835,6 +837,8 @@ Node.js连接器支持的系统有： | **OS类型** | Linux | Win64 | Win32 | Linux | Linux | | **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** | +Node.js连接器的使用参见视频教程 + ### 安装准备 * 应用驱动安装请参考安装连接器驱动步骤。 @@ -909,7 +913,7 @@ node nodejsChecker.js host=localhost #### 建立连接 -使用node.js连接器时，必须先require ```td2.0-connector```，然后使用 ```taos.connect``` 函数。```taos.connect``` 函数必须提供的参数是```host```，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化```cursor``` 来和TDengine服务端通信 +使用node.js连接器时，必须先require `td2.0-connector`，然后使用 `taos.connect` 函数。`taos.connect` 函数必须提供的参数是`host`，其它参数在没有提供的情况下会使用如下的默认值。最后需要初始化`cursor` 来和TDengine服务端通信 ```javascript const taos = require('td2.0-connector'); @@ -945,13 +949,13 @@ TDengine目前还不支持update和delete语句。 #### 查询 -可通过 ```cursor.query``` 函数来查询数据库。 +可通过 `cursor.query` 函数来查询数据库。 ```javascript var query = cursor.query('show databases;') ``` -查询的结果可以通过 ```query.execute()``` 函数获取并打印出来 +查询的结果可以通过 `query.execute()` 函数获取并打印出来 ```javascript var promise = query.execute(); @@ -959,7 +963,7 @@ promise.then(function(result) { result.pretty(); }); ``` -格式化查询语句还可以使用```query```的```bind```方法。如下面的示例：```query```会自动将提供的数值填入查询语句的```?```里。 +格式化查询语句还可以使用`query`的`bind`方法。如下面的示例：`query`会自动将提供的数值填入查询语句的`?`里。 ```javascript var query = cursor.query('select * from meterinfo.meters where ts <= ? and areaid = ?;').bind(new Date(), 5); @@ -967,7 +971,7 @@ query.execute().then(function(result) { result.pretty(); }) ``` -如果在```query```语句里提供第二个参数并设为```true```也可以立即获取查询结果。如下： +如果在`query`语句里提供第二个参数并设为`true`也可以立即获取查询结果。如下： ```javascript var promise = cursor.query('select * from meterinfo.meters where v1 = 30;', true) @@ -991,4 +995,4 @@ promise2.then(function(result) { ### 示例 这里提供了一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例 -这里同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`. \ No newline at end of file +这里同样是一个使用NodeJS 连接器建表，插入天气数据并查询插入的数据的代码示例，但和上面不同的是，该示例只使用`cursor`. diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/webdocs/markdowndocs/connector-java-ch.md index fe029ba269c3b3784ae50f7d5a8708da9aa7d426..7ba573d2e44bb3848d13a2f6b7cb81038843a291 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-java-ch.md @@ -6,6 +6,8 @@ Java连接器支持的系统有： | **OS类型** | Linux | Win64 | Win32 | Linux | Linux | | **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** | +Java连接器的使用请参见视频教程。 + TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。 由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 8c2ef193826ca6b433b0923b76461527cd436400..73fa915abdde8e0940f142fc56124f66e32a6d25 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -39,10 +39,10 @@ # number of management nodes in the system # numOfMnodes 3 -# enable/disable backuping vnode directory when removing dnode +# enable/disable backuping vnode directory when removing vnode # vnodeBak 1 -# if report installation / use information +# enable/disable installation / usage report # telemetryReporting 1 # enable/disable load balancing @@ -81,7 +81,7 @@ # minimum time window, milli-second # minIntervalTime 10 -# maximum delay before launching a stream compution, milli-second +# maximum delay before launching a stream computation, milli-second # maxStreamCompDelay 20000 # maximum delay before launching a stream computation for the first time, milli-second @@ -156,7 +156,7 @@ # max number of connections allowed in dnode # maxShellConns 5000 -# max numerber of connections allowed in client +# max number of connections allowed in client # maxConnections 5000 # stop writing logs when the disk size of the log folder is less than this value @@ -187,7 +187,7 @@ # restfulRowLimit 10240 # The following parameter is used to limit the maximum number of lines in log files. -# max number of rows per log filters +# max number of lines per log filters # numOfLogLines 10000000 # enable/disable async log @@ -199,7 +199,9 @@ # The following parameters are used for debug purpose only. # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log. +# 131: output warning and error +# 135: output debug, warning and error +# 143: output trace, debug, warning and error to log # 199: output debug, warning and error to both screen and file # 207: output trace, debug, warning and error to both screen and file @@ -231,10 +233,10 @@ # cDebugFlag 131 # debug flag for JNI -# jniDebugflag 131 +# jniDebugFlag 131 # debug flag for storage -# uDebugflag 131 +# uDebugFlag 131 # debug flag for http server # httpDebugFlag 131 @@ -243,12 +245,12 @@ # monDebugFlag 131 # debug flag for query -# qDebugflag 131 +# qDebugFlag 131 # debug flag for vnode -# vDebugflag 131 +# vDebugFlag 131 -# debug flag for http server +# debug flag for TSDB # tsdbDebugFlag 131 # debug flag for continue query diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index d24502a1cb8e69ddaf3989a89e51cc07dfb55f00..3d57ece2adf4e8efd3338f52787d5d9424016e78 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -26,6 +26,7 @@ else ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index edc7de96923266729d1985cb43abfa3003f9178e..431093be950e2c2bb199ed294b57641290fe6221 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/taosdemox ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 8c23ab802d54088dafc4b4c6a5d1b6241c881f2e..6f012aa80eac67ba42e7ca561c95ef6d905c2100 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosdemox %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include @@ -135,7 +136,8 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - #${csudo} rm -f ${bin_link_dir}/taosdump || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 6f481a6d6c425393ccd05019c8cbcf0614392e4a..338abcc6a05603e3ce313fc615a6240754e5315a 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -175,6 +175,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : @@ -186,6 +187,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 0467300953ca34d1f9b24ba9999ccd156a0d8d09..dd116e9bfb0472b661321e9af786521300ed2f74 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -86,6 +86,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : fi ${csudo} rm -f ${bin_link_dir}/rmtaos || : @@ -97,6 +98,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 26977e12f4b7ee780948399605f28603817f1aa2..04fd23d5abb2874d0b3a68b937be529bd2c91a59 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -85,8 +85,9 @@ function install_bin() { # Remove links ${csudo} rm -f ${bin_link_dir}/power || : if [ "$osType" != "Darwin" ]; then - ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdump || : + ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdemox || : + ${csudo} rm -f ${bin_link_dir}/powerdump || : fi ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -97,6 +98,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : + [ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || : [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || : fi [ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || : diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index 1e3cb81b7d263149767d50a4a8f76b64736e7595..28788ceb74ac553da4f72a405085cfe264e47fa2 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -174,6 +174,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerd || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdemox || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -184,6 +185,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : [ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd || : [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : + [ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || : [ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index ee79a560407c650de8c511964f611cf8cacfd2d5..00dfcb75590a50d23180e5eb2b6ad38f28924bba 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index fdb3e0e5cc8c7add9c8ea8cdc9224964b3c80153..509df31297d5b442a05516b37a1b82572ae4f610 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -77,6 +77,7 @@ if [ "$osType" != "Darwin" ]; then cp ${build_dir}/bin/taos ${install_dir}/bin/power cp ${script_dir}/remove_power.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo + cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 5ae5cbbcdc00c1fd862501a6eb0dccd3ac0fa007..0b4659a9112f933a5ba9b91f52f3df203d18a2e7 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index 13849484695f98a5fc206d88ec6a5de0cc930a9b..ba57fe5e9687b32b73a0a3ff4d48807f04a0202d 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -77,6 +77,7 @@ else cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd cp ${script_dir}/remove_power.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo + cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${build_dir}/bin/tarbitrator ${install_dir}/bin cp ${script_dir}/set_core.sh ${install_dir}/bin diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 6b4d1837647ee78747e271ddcdf5fac95d490b10..9d15b9736ed8647440ee3cd0f25276d635da1952 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -96,6 +96,8 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdemox || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -105,6 +107,8 @@ function install_bin() { [ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : [ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : [ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${bin_dir}/taosdemox ] && ${csudo} ln -s ${bin_dir}/taosdemox ${bin_link_dir}/taosdemox || : + [ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : } @@ -265,8 +269,14 @@ function install_config() { [ -f ${cfg_dir}/taos.cfg ] && ${csudo} cp ${cfg_dir}/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi + + # Save standard input to 6 and open / dev / TTY on standard input + exec 6<&0 0 /dev/null; then csudo="sudo" fi -#ulimit -c unlimited +if [[ ! -n ${corePath} ]]; then + echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:" + read corePath + while true; do + if [[ ! -z "$corePath" ]]; then + break + else + read -p "Please enter a file directory to save the coredump file:" corePath + fi + done +fi + +ulimit -c unlimited ${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||: ${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||: source /etc/profile -${csudo} mkdir -p /coredump ||: -${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p' ||: -${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern ||: +${csudo} mkdir -p ${corePath} ||: +${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p ||: +${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern ||: diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 0b9c33950c9c1f0e67382ace4d8a962da50ac509..c4b2039737723ca7704cb8a40a3f8d1e128df008 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.11.0' +version: '2.0.13.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.11.0 + - usr/lib/libtaos.so.2.0.13.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 2532cf09a76de868aa97f5d6a7889b2f174705c8..3e1d5eda763c99afb8a0c5dcd3f08996eec80337 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -66,6 +66,77 @@ static bool bnCheckFree(SDnodeObj *pDnode) { return true; } +static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { + SVnodeGid tmp = *pVnodeGid1; + *pVnodeGid1 = *pVnodeGid2; + *pVnodeGid2 = tmp; +} + +static void bnAdjustVnodeIndex(SVgObj *pInVg) { + int32_t d0Id = pInVg->vnodeGid[0].dnodeId; + int32_t d1Id = pInVg->vnodeGid[1].dnodeId; + int32_t d2Id = pInVg->vnodeGid[2].dnodeId; + + int32_t vgId = pInVg->vgId; + int32_t d0Num = 0; + int32_t d1Num = 0; + int32_t d2Num = 0; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + + if (pVgroup->vgId != vgId) { + if (pVgroup->vnodeGid[0].dnodeId == d0Id) d0Num++; + if (pVgroup->vnodeGid[0].dnodeId == d1Id) d1Num++; + if (pVgroup->vnodeGid[0].dnodeId == d2Id) d2Num++; + } + + mnodeDecVgroupRef(pVgroup); + } + + if (pInVg->numOfVnodes == 1) { + } + + if (pInVg->numOfVnodes == 2) { + mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num); + if (d0Num > d1Num) { + mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); + } + } + + if (pInVg->numOfVnodes >= 3) { + mDebug("vgId:%d, dnode:%d num:%d dnode:%d num:%d dnode:%d num:%d", pInVg->vgId, d0Id, d0Num, d1Id, d1Num, d2Id, d2Num); + if (d0Num <= d1Num && d0Num <= d2Num) { + if (d1Num > d2Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } else if (d1Num <= d2Num && d1Num <= d0Num) { + mDebug("vgId:%d, adjust vnode index 0 to 1", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[1]); + if (d0Num > d2Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } else { + mDebug("vgId:%d, adjust vnode index 0 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[0], &pInVg->vnodeGid[2]); + if (d1Num > d0Num) { + mDebug("vgId:%d, adjust vnode index 1 to 2", pInVg->vgId); + bnSwapVnodeGid(&pInVg->vnodeGid[1], &pInVg->vnodeGid[2]); + } + } + } + + for (int i = 0; i < pInVg->numOfVnodes; ++i) { + mDebug("vgId:%d index:%d dnodeId:%d", pInVg->vgId, i, pInVg->vnodeGid[i].dnodeId); + } +} + static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId); @@ -88,15 +159,10 @@ static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); pVgroup->numOfVnodes = numOfVnodes; + bnAdjustVnodeIndex(pVgroup); mnodeUpdateVgroup(pVgroup); } -static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { - SVnodeGid tmp = *pVnodeGid1; - *pVnodeGid1 = *pVnodeGid2; - *pVnodeGid2 = tmp; -} - int32_t bnAllocVnodes(SVgObj *pVgroup) { int32_t dnode = 0; int32_t vnodes = 0; @@ -147,6 +213,7 @@ int32_t bnAllocVnodes(SVgObj *pVgroup) { } } + bnAdjustVnodeIndex(pVgroup); bnReleaseDnodes(); bnUnLock(); return TSDB_CODE_SUCCESS; @@ -157,17 +224,32 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { return false; } + int32_t rmVnodeVer = 0; + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SVnodeGid *pVnode = pVgroup->vnodeGid + i; + if (pVnode == pRmVnode) { + rmVnodeVer = mnodeGetVgidVer(pVnode->vver); + mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i, + pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer); + } + } + bool isReady = false; for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVnode = pVgroup->vnodeGid + i; + SDnodeObj *pDnode = pVnode->pDnode; if (pVnode == pRmVnode) continue; + int32_t vver = mnodeGetVgidVer(pVnode->vver); - mTrace("vgId:%d, check vgroup status, dnode:%d status:%d, vnode role:%s", pVgroup->vgId, pVnode->pDnode->dnodeId, - pVnode->pDnode->status, syncRole[pVnode->role]); - if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue; - if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; + mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d", pVgroup->vgId, i, + pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); + if (pDnode->status == TAOS_DN_STATUS_DROPPING) continue; + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; + if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue; - if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) { + if (rmVnodeVer == 0 || vver >= rmVnodeVer) { + mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d", + pVgroup->vgId, i, pVnode->dnodeId, dnodeStatus[pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer); isReady = true; } } @@ -189,7 +271,7 @@ static int32_t bnRemoveVnode(SVgObj *pVgroup) { mDebug("vgId:%d, is not ready", pVgroup->vgId); return -1; } else { - mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); + mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); bnDiscardVnode(pVgroup, pSelVnode); return TSDB_CODE_SUCCESS; } @@ -233,6 +315,7 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes vnodeGids[numOfVnodes].pDnode = pDestDnode; numOfVnodes++; + // move the src vnode to the end for (int32_t v = 0; v < numOfVnodes; ++v) { if (pSrcDnode != NULL && pSrcDnode->dnodeId == vnodeGids[v].dnodeId) { bnSwapVnodeGid(&vnodeGids[v], &vnodeGids[numOfVnodes - 1]); @@ -241,6 +324,11 @@ static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDes } } + // adjust the vgroup postion + if (pSrcDnode == NULL) { + bnAdjustVnodeIndex(pVgroup); + } + memcpy(&pVgroup->vnodeGid, &vnodeGids, sizeof(SVnodeGid) * TSDB_MAX_REPLICA); pVgroup->numOfVnodes = numOfVnodes; atomic_add_fetch_32(&pDestDnode->openVnodes, 1); @@ -330,7 +418,7 @@ void bnReset() { tsAccessSquence = 0; } -static int32_t bnMonitorVgroups() { +static bool bnMonitorVgroups() { void * pIter = NULL; SVgObj *pVgroup = NULL; bool hasUpdatingVgroup = false; @@ -489,6 +577,7 @@ void bnCheckStatus() { mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence, pDnode->lastAccess, pDnode->status); bnSetVgroupOffline(pDnode); + bnStartTimer(3000); } } mnodeDecDnodeRef(pDnode); diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index bf046a9faea6e9e3d7b324e7786c594d16a9aeb7..84f8694fca7248abb27529f5e8268dc0e08bf815 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -31,7 +31,10 @@ static void *bnThreadFunc(void *arg) { } pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); + mDebug("balance thread wakes up to work"); bool updateSoon = bnStart(); + mDebug("balance thread finished this poll, updateSoon:%d", updateSoon); + bnStartTimer(updateSoon ? 1000 : -1); pthread_mutex_unlock(&(tsBnThread.mutex)); } @@ -101,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) { tsBnThread.timer = NULL; tsAccessSquence++; - bnCheckStatus(); bnStartTimer(-1); + bnCheckStatus(); if (handle == NULL) { if (tsAccessSquence % tsBalanceInterval == 0) { @@ -121,6 +124,7 @@ void bnStartTimer(int64_t mseconds) { bool updateSoon = (mseconds != -1); if (updateSoon) { + mTrace("balance function will be called after %" PRId64 " ms", mseconds); taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 43ba31f331fc5c92d4142a008db458b23395a473..06176451881b30ebe4c24bfcd478ceeb221a86de 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -38,12 +38,6 @@ typedef struct SLocalDataSource { tFilePage filePage; } SLocalDataSource; -enum { - TSC_LOCALREDUCE_READY = 0x0, - TSC_LOCALREDUCE_IN_PROGRESS = 0x1, - TSC_LOCALREDUCE_TOBE_FREED = 0x2, -}; - typedef struct SLocalReducer { SLocalDataSource ** pLocalDataSrc; int32_t numOfBuffer; @@ -56,7 +50,6 @@ typedef struct SLocalReducer { tFilePage * pTempBuffer; struct SQLFunctionCtx *pCtx; int32_t rowSize; // size of each intermediate result. - int32_t status; // denote it is in reduce process, in reduce process, it bool hasPrevRow; // cannot be released bool hasUnprocessedRow; tOrderDescriptor * pDesc; diff --git a/src/client/inc/tscLog.h b/src/client/inc/tscLog.h index 5bbf05aa5f11eedc81a34f4a3d0934774db386bf..5273a87ea0d0549420acf2b6679f50ce22159ebc 100644 --- a/src/client/inc/tscLog.h +++ b/src/client/inc/tscLog.h @@ -22,8 +22,8 @@ extern "C" { #include "tlog.h" -extern uint32_t cDebugFlag; -extern uint32_t tscEmbedded; +extern int32_t cDebugFlag; +extern int8_t tscEmbedded; #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 37d05de731d88d6f7e8522349976221176b3199d..8d82c65cee5cb26c3a4a37e6bbe6076dae28aaa5 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -20,9 +20,6 @@ extern "C" { #endif -/* - * @date 2018/09/30 - */ #include "exception.h" #include "os.h" #include "qExtbuffer.h" @@ -216,7 +213,7 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); @@ -276,7 +273,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreClauseToTry(SSqlObj* pSql); -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache); +void tscFreeQueryInfo(SSqlCmd* pCmd); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -290,6 +287,14 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ char* serializeTagData(STagData* pTagData, char* pMsg); int32_t copyTagData(STagData* dst, const STagData* src); +STableMeta* createSuperTableMeta(STableMetaMsg* pChild); +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); +uint32_t tscGetTableMetaMaxSize(); +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name); +STableMeta* tscTableMetaClone(STableMeta* pTableMeta); + + void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); char* strdup_throw(const char* str); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index f6dc45398f35c38598f3f3132b2f6f5601a4ed68..7c41164a046165c785d3d4e0e0a8979854e4b6f2 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -105,7 +105,10 @@ SSchema tscGetTbnameColumnSchema(); * @param size size of the table meta * @return */ -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg); + +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src); +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 99ee62fa7fd3eb798a17b38d4726be94f664f9ba..25a299d0988144980dded10b170cbe7b7c662b04 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -56,22 +56,28 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; -typedef struct SCorVgroupInfo { - int32_t version; - int8_t inUse; - int8_t numOfEps; - SEpAddr1 epAddr[TSDB_MAX_REPLICA]; -} SCorVgroupInfo; +typedef struct SNewVgroupInfo { + int32_t vgId; + int8_t inUse; + int8_t numOfEps; + SEpAddrMsg ep[TSDB_MAX_REPLICA]; +} SNewVgroupInfo; + +typedef struct CChildTableMeta { + int32_t vgId; + STableId id; + uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; +} CChildTableMeta; typedef struct STableMeta { - STableComInfo tableInfo; + int32_t vgId; + STableId id; uint8_t tableType; + char sTableName[TSDB_TABLE_FNAME_LEN]; int16_t sversion; int16_t tversion; - char sTableId[TSDB_TABLE_FNAME_LEN]; - SVgroupInfo vgroupInfo; - SCorVgroupInfo corVgroupInfo; - STableId id; + STableComInfo tableInfo; SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; @@ -170,7 +176,7 @@ typedef struct SParamInfo { } SParamInfo; typedef struct STableDataBlocks { - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableName[TSDB_TABLE_FNAME_LEN]; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id @@ -248,7 +254,7 @@ typedef struct { int8_t submitSchema; // submit block is built with table schema STagData tagData; // NOTE: pTagData->data is used as a variant length array - STableMeta **pTableMetaList; // all involved tableMeta list of current insert sql statement. + char **pTableNameList; // all involved tableMeta list of current insert sql statement. int32_t numOfTables; SHashObj *pTableBlockHashList; // data block for each table @@ -307,7 +313,7 @@ typedef struct STscObj { SRpcCorEpSet *tscCorMgmtEpSet; void* pDnodeConn; pthread_mutex_t mutex; - T_REF_DECLARE() + int32_t numOfObj; // number of sqlObj from this tscObj } STscObj; typedef struct SSubqueryState { @@ -385,7 +391,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); int tscProcessSql(SSqlObj *pSql); int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex); -void tscQueueAsyncRes(SSqlObj *pSql); +void tscAsyncResultOnError(SSqlObj *pSql); void tscQueueAsyncError(void(*fp), void *param, int32_t code); @@ -399,7 +405,7 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); +void tscResetSqlCmdObj(SSqlCmd *pCmd); /** * free query result of the sql object @@ -413,14 +419,13 @@ void tscFreeSqlResult(SSqlObj *pSql); */ void tscFreeSqlObj(SSqlObj *pSql); void tscFreeRegisteredSqlObj(void *pSql); -void tscFreeTableMetaHelper(void *pTableMeta); void tscCloseTscObj(void *pObj); // todo move to taos? or create a new file: taos_internal.h TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos); -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res); +TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res); void waitForQueryRsp(void *param, TAOS_RES *tres, int code); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen); @@ -478,15 +483,16 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } } -extern SCacheObj* tscMetaCache; -extern int tscObjRef; -extern void * tscTmr; -extern void * tscQhandle; -extern int tscKeepConn[]; -extern int tsInsertHeadSize; -extern int tscNumOfThreads; -extern int tscRefId; - +extern int32_t sentinel; +extern SHashObj *tscVgroupMap; +extern SHashObj *tscTableMetaInfo; + +extern int tscObjRef; +extern void *tscTmr; +extern void *tscQhandle; +extern int tscKeepConn[]; +extern int tscRefId; +extern int tscNumOfObj; // number of existed sqlObj in current process. extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 97b962e53acc27f5a5f5e09adf97dc9bc0ef1801..96aeb9d60de1ab6fbaeebcb54e2da1ab316179f8 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -18,7 +18,6 @@ #include "tnote.h" #include "trpc.h" -#include "tcache.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscLocalMerge.h" @@ -57,7 +56,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (pSql->sqlstr == NULL) { tscError("%p failed to malloc sql string buffer", pSql); pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -71,7 +70,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -166,7 +165,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pRes->code = numOfRows; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -217,7 +216,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; pSql->param = param; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -280,7 +279,7 @@ void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), pSql->param = param; pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -382,16 +381,16 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) { } -void tscQueueAsyncRes(SSqlObj *pSql) { +void tscAsyncResultOnError(SSqlObj *pSql) { if (pSql == NULL || pSql->signature != pSql) { tscDebug("%p SqlObj is freed, not add into queue async res", pSql); return; } + assert(pSql->res.code != TSDB_CODE_SUCCESS); tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code)); SSqlRes *pRes = &pSql->res; - if (pSql->fp == NULL || pSql->fetchFp == NULL){ return; } @@ -423,7 +422,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { // check if it is a sub-query of super table query first, if true, enter another routine if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) { - tscDebug("%p update table meta in local cache, continue to process sql and send the corresponding query", pSql); + tscDebug("%p update local table meta, continue to process sql and send the corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -440,7 +439,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { return; } else { // continue to process normal async query if (pCmd->parseFinished) { - tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql); + tscDebug("%p update local table meta, continue to process sql and send corresponding query", pSql); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); code = tscGetTableMeta(pSql, pTableMetaInfo); @@ -455,7 +454,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pCmd->command == TSDB_SQL_SELECT) { tscDebug("%p redo parse sql string and proceed", pSql); pCmd->parseFinished = false; - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmdObj(pCmd); code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -532,6 +531,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { _error: if (code != TSDB_CODE_SUCCESS) { pSql->res.code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 192af4dbdfea787a157ff2c03a0c4857128dede2..16bbd420c0cd77bd3f94ce64d840ccda36a36eb6 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -17,7 +17,6 @@ #include "taosmsg.h" #include "taosdef.h" -#include "tcache.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" @@ -273,7 +272,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code != TSDB_CODE_SUCCESS) { taos_free_result(pSql); free(builder); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -291,7 +290,7 @@ void tscSCreateCallBack(void *param, TAOS_RES *tres, int code) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, code); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -571,7 +570,7 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0}; extractDBName(pTableMetaInfo->name, fullName); - extractTableName(pMeta->sTableId, param->sTableName); + extractTableName(pMeta->sTableName, param->sTableName); snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName); extractTableName(pTableMetaInfo->name, param->buf); @@ -901,7 +900,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { pRes->code = tscProcessShowCreateDatabase(pSql); } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosCacheEmpty(tscMetaCache); + taosHashEmpty(tscTableMetaInfo); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); @@ -925,7 +924,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { (*pSql->fp)(pSql->param, pSql, code); } else if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS){ } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index a99918975e11a1f51bc93eebf6053cfd8dca05c3..921aa9bade64759d149769726db0a5381c4b112c 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -89,11 +89,11 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->startOffset = 0; pCtx->size = 1; pCtx->hasNull = true; - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; // for top/bottom function, the output of timestamp is the first column int32_t functionId = pExpr->functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; @@ -493,13 +493,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { // there is no more result, so we release all allocated resource SLocalReducer *pLocalReducer = (SLocalReducer *)atomic_exchange_ptr(&pRes->pLocalReducer, NULL); if (pLocalReducer != NULL) { - int32_t status = 0; - while ((status = atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, - TSC_LOCALREDUCE_TOBE_FREED)) == TSC_LOCALREDUCE_IN_PROGRESS) { - taosMsleep(100); - tscDebug("%p waiting for delete procedure, status: %d", pSql, status); - } - pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo); if (pLocalReducer->pCtx != NULL) { @@ -911,6 +904,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, } } + if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) { + pRes->numOfRows = 0; + pBeforeFillData->num = 0; + pLocalReducer->discard = true; + return; + } + pRes->numOfRowsGroup += pRes->numOfRows; // impose the limitation of output rows on the final result @@ -1067,7 +1067,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, pCtx->param[0].i64Key = pExpr->param[0].i64Key; } - pCtx->currentStage = SECONDARY_STAGE_MERGE; + pCtx->currentStage = MERGE_STAGE; if (needInit) { aAggs[pCtx->functionId].init(pCtx); @@ -1080,7 +1080,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, continue; } - aAggs[functionId].distSecondaryMergeFunc(&pLocalReducer->pCtx[j]); + aAggs[functionId].mergeFunc(&pLocalReducer->pCtx[j]); } } @@ -1296,6 +1296,10 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// re for (int32_t i = 0; i < t; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity; + + if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM || pExpr->functionId == TSDB_FUNC_DIFF) { + pLocalReducer->pCtx[i].ptsOutputBuf = pLocalReducer->pCtx[0].aOutputBuf; + } } memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage)); @@ -1430,24 +1434,13 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { SLocalReducer *pLocalReducer = pRes->pLocalReducer; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - // set the data merge in progress - int32_t prevStatus = - atomic_val_compare_exchange_32(&pLocalReducer->status, TSC_LOCALREDUCE_READY, TSC_LOCALREDUCE_IN_PROGRESS); - if (prevStatus != TSC_LOCALREDUCE_READY) { - assert(prevStatus == TSC_LOCALREDUCE_TOBE_FREED); // it is in tscDestroyLocalReducer function already - return TSDB_CODE_SUCCESS; - } - - tFilePage *tmpBuffer = pLocalReducer->pTempBuffer; + tFilePage *tmpBuffer = pLocalReducer->pTempBuffer; if (doHandleLastRemainData(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } if (doBuildFilledResultForGroup(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } @@ -1503,7 +1496,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { pLocalReducer->discardData->num = 0; if (saveGroupResultInfo(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; return TSDB_CODE_SUCCESS; } @@ -1549,7 +1541,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { // here we do not check the return value adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); - assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS); if (pRes->numOfRows == 0) { handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); @@ -1560,7 +1551,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * If previous group is not skipped, keep it in pRes->numOfGroups */ if (notSkipped && saveGroupResultInfo(pSql)) { - pLocalReducer->status = TSC_LOCALREDUCE_READY; return TSDB_CODE_SUCCESS; } @@ -1580,7 +1570,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { if (pRes->numOfRows == 0) { continue; } else { - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. return TSDB_CODE_SUCCESS; } } else { // result buffer is not full @@ -1605,9 +1594,6 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { genFinalResults(pSql, pLocalReducer, true); } - assert(pLocalReducer->status == TSC_LOCALREDUCE_IN_PROGRESS && pRes->row == 0); - pLocalReducer->status = TSC_LOCALREDUCE_READY; // set the flag, taos_free_result can release this result. - return TSDB_CODE_SUCCESS; } @@ -1661,7 +1647,7 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ // calculate the result from several other columns if (pSup->pArithExprInfo != NULL) { arithSup.pArithExpr = pSup->pArithExprInfo; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); } else { SSqlExpr* pExpr = pSup->pSqlExpr; memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num)); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ded12f64eae057519002a0a3547fe672ffbb43e1..ec90d21394a5dce657652fb9e6c2ca81dd2f120f 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -731,7 +731,7 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColI return code; } - dataBuf->vgId = pTableMeta->vgroupInfo.vgId; + dataBuf->vgId = pTableMeta->vgId; dataBuf->numOfTables = 1; *totalNum += numOfRows; @@ -1339,7 +1339,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { free(sqlstr); } else { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmdObj(pCmd); free(pSql->sqlstr); pSql->sqlstr = sqlstr; pSql->parseRetry++; @@ -1351,7 +1351,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) { - tscResetSqlCmdObj(pCmd, true); + tscResetSqlCmdObj(pCmd); pSql->parseRetry++; ret = tscToSQLCmd(pSql, &SQLInfo); } @@ -1413,13 +1413,25 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // handle error assert(taos_errno(pSql) == code); - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); + do { + if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + assert(pSql->res.numOfRows == 0); + int32_t errc = fseek(fp, 0, SEEK_SET); + if (errc < 0) { + tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno)); + } else { + break; + } + } - pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); - return; + taos_free_result(pSql); + tfree(pSupporter); + fclose(fp); + + pParentSql->res.code = code; + tscAsyncResultOnError(pParentSql); + return; + } while (0); } // accumulate the total submit records @@ -1439,7 +1451,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { int32_t count = 0; int32_t maxRows = 0; - tfree(pCmd->pTableMetaList); + tfree(pCmd->pTableNameList); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); if (pCmd->pTableBlockHashList == NULL) { @@ -1488,7 +1500,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { code = doPackSendDataBlock(pSql, count, pTableDataBlock); if (code != TSDB_CODE_SUCCESS) { pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1523,7 +1535,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); tfree(pSupporter); - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 71863cbe1561919070cd766423eae3f99081b7e0..6479a7ecba505c90025b66c41f8740651132cb66 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -75,11 +75,11 @@ static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SC static int32_t convertFunctionId(int32_t optr, int16_t* functionId); static uint8_t convertOptr(SStrToken *pToken); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool intervalQuery); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); @@ -666,6 +666,7 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; const char* msg3 = "sliding cannot be used without interval"; + const char* msg4 = "top/bottom query does not support order by value in interval query"; SSqlCmd* pCmd = &pSql->cmd; @@ -712,6 +713,11 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } + int32_t colId = pQueryInfo->order.orderColId; + if (pQueryInfo->interval.interval > 0 && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + return TSDB_CODE_SUCCESS; } @@ -904,7 +910,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa * that are corresponding to the old name for the new table name. */ if (strlen(oldName) > 0 && strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearTableMetaInfo(pTableMetaInfo, false); + tscClearTableMetaInfo(pTableMetaInfo); } return TSDB_CODE_SUCCESS; @@ -996,33 +1002,6 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC return false; } - int32_t nLen = 0; - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - if (p->bytes == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; - } - - nLen += p->bytes; - } - - // max tag row length must be less than TSDB_MAX_TAGS_LEN - if (nLen > TSDB_MAX_TAGS_LEN) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; - } - - // field name must be unique - for (int32_t i = 0; i < numOfTags; ++i) { - TAOS_FIELD* p = taosArrayGet(pTagsList, i); - - if (has(pFieldList, 0, p->name) == true) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; - } - } - /* timestamp in tag is not allowed */ for (int32_t i = 0; i < numOfTags; ++i) { TAOS_FIELD* p = taosArrayGet(pTagsList, i); @@ -1054,6 +1033,33 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC } } + int32_t nLen = 0; + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + if (p->bytes == 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + return false; + } + + nLen += p->bytes; + } + + // max tag row length must be less than TSDB_MAX_TAGS_LEN + if (nLen > TSDB_MAX_TAGS_LEN) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + return false; + } + + // field name must be unique + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + + if (has(pFieldList, 0, p->name) == true) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return false; + } + } + return true; } @@ -1469,7 +1475,7 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery) { assert(pSelection != NULL && pCmd != NULL); const char* msg2 = "functions can not be mixed up"; @@ -1525,7 +1531,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel addPrimaryTsColIntoResult(pQueryInfo); } - if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { + if (!functionCompatibleCheck(pQueryInfo, joinQuery, intervalQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2804,7 +2810,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool intervalQuery) { int32_t startIdx = 0; size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); @@ -2820,6 +2826,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t factor = functionCompatList[tscSqlExprGet(pQueryInfo, startIdx)->functionId]; + if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { + return false; + } + // diff function cannot be executed with other function // arithmetic function can be executed with other arithmetic functions size_t size = tscSqlExprNumOfExprs(pQueryInfo); @@ -2844,7 +2854,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { } } - if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { + if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { return false; } } @@ -4013,6 +4023,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, if (pExpr->nSQLOptr == TK_LIKE) { char* str = taosStringBuilderGetResult(sb, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); return TSDB_CODE_SUCCESS; } @@ -4062,6 +4073,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* str = taosStringBuilderGetResult(&sb1, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); + pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); taosStringBuilderDestroy(&sb1); tfree(segments); @@ -4472,6 +4484,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery const char* msg = "illegal value or data overflow"; const char* msg1 = "value is expected"; const char* msg2 = "invalid fill option"; + const char* msg3 = "top/bottom not support fill"; if (pItem->pVar.nType != TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); @@ -4554,6 +4567,14 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + return TSDB_CODE_SUCCESS; } @@ -4646,7 +4667,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu if (!(orderByTags || orderByTS) && !isTopBottomQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } else { + } else { // order by top/bottom result value column is not supported in case of interval query. assert(!(orderByTags && orderByTS)); } @@ -4936,7 +4957,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId); + pUpdateMsg->head.vgId = htonl(pTableMeta->vgId); pUpdateMsg->tid = htonl(pTableMeta->id.tid); pUpdateMsg->uid = htobe64(pTableMeta->id.uid); pUpdateMsg->colId = htons(pTagsSchema->colId); @@ -5442,6 +5463,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->quorum = pCreateDb->quorum; pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->update = pCreateDb->update; + pMsg->cacheLastRow = pCreateDb->cachelast; } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { @@ -6302,7 +6324,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6365,6 +6387,41 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_SUCCESS; } +static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { + const char* msg3 = "start(end) time of query range required or time range too large"; + + if (pQueryInfo->interval.interval == 0) { + return TSDB_CODE_SUCCESS; + } + + bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); + if (initialWindows) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); + + int64_t intervalRange = 0; + if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { + int64_t f = 1; + if (pQueryInfo->interval.intervalUnit == 'n') { + f = 30L * MILLISECOND_PER_DAY; + } else if (pQueryInfo->interval.intervalUnit == 'y') { + f = 365L * MILLISECOND_PER_DAY; + } + + intervalRange = pQueryInfo->interval.interval * f; + } else { + intervalRange = pQueryInfo->interval.interval; + } + // number of result is not greater than 10,000,000 + if ((timeRange == 0) || (timeRange / intervalRange) >= MAX_INTERVAL_TIME_WINDOW) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + return TSDB_CODE_SUCCESS; +} + int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0)); @@ -6512,7 +6569,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { int32_t joinQuery = (pQuerySql->from != NULL && taosArrayGetSize(pQuerySql->from) > 2); - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { + int32_t intervalQuery = !(pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0); + + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery, intervalQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6560,31 +6619,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { tscFieldInfoUpdateOffset(pQueryInfo); - /* - * fill options are set at the end position, when all columns are set properly - * the columns may be increased due to group by operation - */ if (pQuerySql->fillType != NULL) { if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (pQueryInfo->interval.interval > 0) { - bool initialWindows = TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER); - if (initialWindows) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } - - int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->interval.interval) > MAX_INTERVAL_TIME_WINDOW) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } + /* + * fill options are set at the end position, when all columns are set properly + * the columns may be increased due to group by operation + */ + if ((code = checkQueryRangeForFill(pCmd, pQueryInfo)) != TSDB_CODE_SUCCESS) { + return code; } - int32_t ret = parseFillClause(pCmd, pQueryInfo, pQuerySql); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if ((code = parseFillClause(pCmd, pQueryInfo, pQuerySql)) != TSDB_CODE_SUCCESS) { + return code; } } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index fcc93ffadc4f7dc62b79fffe245947799a770b40..b9d38e3ea76d9ba15c7fe5eb5906e205423aade5 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -130,23 +130,15 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) { return NULL; } -static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) { - corVgroupInfo->version = 0; - corVgroupInfo->inUse = 0; - corVgroupInfo->numOfEps = vgroupInfo->numOfEps; - for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) { - corVgroupInfo->epAddr[i].fqdn = strdup(vgroupInfo->epAddr[i].fqdn); - corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port; - } -} - -STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { - assert(pTableMetaMsg != NULL); +STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2 && pTableMetaMsg->numOfTags >= 0); int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); + pTableMeta->tableType = pTableMetaMsg->tableType; - + pTableMeta->vgId = pTableMetaMsg->vgroup.vgId; + pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, .precision = pTableMetaMsg->precision, @@ -156,22 +148,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->id.tid = pTableMetaMsg->tid; pTableMeta->id.uid = pTableMetaMsg->uid; - SVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; - pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps; - pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId; - - for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - SEpAddrMsg* pEpMsg = &pTableMetaMsg->vgroup.epAddr[i]; - - pVgroupInfo->epAddr[i].fqdn = strndup(pEpMsg->fqdn, tListLen(pEpMsg->fqdn)); - pVgroupInfo->epAddr[i].port = pEpMsg->port; - } - - tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo); - pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; - tstrncpy(pTableMeta->sTableId, pTableMetaMsg->sTableId, TSDB_TABLE_FNAME_LEN); + tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN); memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); @@ -180,13 +159,44 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - if (size != NULL) { - *size = sizeof(STableMeta) + schemaSize; - } - return pTableMeta; } +bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) { + assert(pExisted != NULL && src != NULL); + if (pExisted->numOfEps != src->numOfEps) { + return false; + } + + for(int32_t i = 0; i < pExisted->numOfEps; ++i) { + if (pExisted->ep[i].port != src->epAddr[i].port) { + return false; + } + + if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) { + return false; + } + } + + return true; +} + +SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { + assert(pVgroupMsg != NULL); + + SNewVgroupInfo info = {0}; + info.numOfEps = pVgroupMsg->numOfEps; + info.vgId = pVgroupMsg->vgId; + info.inUse = 0; + + for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) { + tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN); + info.ep[i].port = pVgroupMsg->epAddr[i].port; + } + + return info; +} + // todo refactor UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ded04388f41d0f783f2379ebf6b09327de4204fb..4949aa9b9dab6efb96f411f433ec420c6fd6d338 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -14,7 +14,6 @@ */ #include "os.h" -#include "tcache.h" #include "tcmdtype.h" #include "trpc.h" #include "tscLocalMerge.h" @@ -45,32 +44,30 @@ static int32_t getWaitingTimeInterval(int32_t count) { return 0; } - return initial * (2<<(count - 2)); + return initial * ((2u)<<(count - 2)); } -static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { - assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); - - SRpcEpSet* pEpSet = &pSql->epSet; +static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupInfo* pVgroupInfo) { + assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); // Issue the query to one of the vnode among a vgroup randomly. // change the inUse property would not affect the isUse attribute of STableMeta pEpSet->inUse = rand() % pVgroupInfo->numOfEps; // apply the FQDN string length check here - bool hasFqdn = false; + bool existed = false; pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; - if (!hasFqdn) { - hasFqdn = (strlen(pEpSet->fqdn[i]) > 0); + int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + if (len > 0) { + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + existed = true; } } - - assert(hasFqdn); + assert(existed); } static void tscDumpMgmtEpSet(SSqlObj *pSql) { @@ -87,7 +84,8 @@ static void tscEpSetHtons(SRpcEpSet *s) { bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; - } + } + for (int32_t i = 0; i < s1->numOfEps; i++) { if (s1->port[i] != s2->port[i] || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) @@ -95,6 +93,7 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { } return true; } + void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { // no need to update if equal SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; @@ -102,37 +101,45 @@ void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { pCorEpSet->epSet = *pEpSet; taosCorEndWrite(&pCorEpSet->version); } -static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { + +static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { if (pVgroupInfo == NULL) { return;} - taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pEpSet->numOfEps = pVgroupInfo->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); + pEpSet->port[i] = pVgroupInfo->ep[i].port; } - taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { + return; + } - taosCorBeginWrite(&pVgroupInfo->version); - tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); - pVgroupInfo->inUse = pEpSet->inUse; - pVgroupInfo->numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - tfree(pVgroupInfo->epAddr[i].fqdn); - pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i])); - pVgroupInfo->epAddr[i].port = pEpSet->port[i]; + int32_t vgId = pTableMetaInfo->pTableMeta->vgId; + if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { + assert(vgId == 0); + return; } - tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); - taosCorEndWrite(&pVgroupInfo->version); + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); + + tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + vgroupInfo.inUse = pEpSet->inUse; + vgroupInfo.numOfEps = pEpSet->numOfEps; + for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { + strncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + vgroupInfo.ep[i].port = pEpSet->port[i]; + } + + tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -304,7 +311,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { return; } - if (pEpSet) { + if (pEpSet) { // todo update this if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if (pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); @@ -438,7 +445,7 @@ int doProcessSql(SSqlObj *pSql) { } if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return pRes->code; } @@ -447,7 +454,7 @@ int doProcessSql(SSqlObj *pSql) { // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. if (code != TSDB_CODE_SUCCESS) { pRes->code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; } @@ -515,8 +522,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); - tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId); + pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); + tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgId); } pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); @@ -535,7 +542,6 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // NOTE: shell message size should not include SMsgDesc int32_t size = pSql->cmd.payloadLen - sizeof(SMsgDesc); - int32_t vgId = pTableMeta->vgroupInfo.vgId; SMsgDesc* pMsgDesc = (SMsgDesc*) pMsg; pMsgDesc->numOfVnodes = htonl(1); // always one vnode @@ -543,7 +549,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(SMsgDesc); SSubmitMsg *pShellMsg = (SSubmitMsg *)pMsg; - pShellMsg->header.vgId = htonl(vgId); + pShellMsg->header.vgId = htonl(pTableMeta->vgId); pShellMsg->header.contLen = htonl(size); // the length not includes the size of SMsgDesc pShellMsg->length = pShellMsg->header.contLen; @@ -551,9 +557,12 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epSet); - tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + + tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, pTableMeta->vgId, pSql->cmd.numOfTablesInSubmit, pSql->epSet.numOfEps); return TSDB_CODE_SUCCESS; } @@ -561,9 +570,11 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { /* * for table query, simply return the size <= 1k */ -static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { +static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + + SSqlCmd* pCmd = &pSql->cmd; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); @@ -571,6 +582,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs * 2); int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; + int32_t tableSerialize = 0; STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -587,7 +600,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) { } return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + - tableSerialize + 4096; + tableSerialize + sqlLen + 4096; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { @@ -597,24 +610,31 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - SVgroupInfo* pVgroupInfo = NULL; + int32_t vgId = -1; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t index = pTableMetaInfo->vgroupIndex; assert(index >= 0); - + + SVgroupInfo* pVgroupInfo = NULL; if (pTableMetaInfo->vgroupList->numOfVgroups > 0) { assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; } + + vgId = pVgroupInfo->vgId; + tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); } else { - pVgroupInfo = &pTableMeta->vgroupInfo; + vgId = pTableMeta->vgId; + + SNewVgroupInfo vgroupInfo = {0}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); } - assert(pVgroupInfo != NULL); + pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - tscSetDnodeEpSet(pSql, pVgroupInfo); - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); + pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->id.tid); @@ -633,7 +653,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo); + tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); @@ -660,7 +680,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - int32_t size = tscEstimateQueryMsgSize(pCmd, pCmd->clauseIndex); + int32_t size = tscEstimateQueryMsgSize(pSql, pCmd->clauseIndex); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); @@ -693,7 +713,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList); - + int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); + if (pQueryInfo->order.order == TSDB_ORDER_ASC) { pQueryMsg->window.skey = htobe64(pQueryInfo->window.skey); pQueryMsg->window.ekey = htobe64(pQueryInfo->window.ekey); @@ -716,10 +737,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); - + pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); + pQueryMsg->sqlstrLen = htonl(sqlLen); + size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number @@ -888,13 +911,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); - *((int16_t *)pMsg) = pCol->colId; + *((int16_t *)pMsg) = htons(pCol->colId); pMsg += sizeof(pCol->colId); - *((int16_t *)pMsg) += pCol->colIndex; + *((int16_t *)pMsg) += htons(pCol->colIndex); pMsg += sizeof(pCol->colIndex); - *((int16_t *)pMsg) += pCol->flag; + *((int16_t *)pMsg) += htons(pCol->flag); pMsg += sizeof(pCol->flag); memcpy(pMsg, pCol->name, tListLen(pCol->name)); @@ -952,13 +975,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += pCond->len; } } - - if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { - *pMsg = 0; - pMsg++; - } else { - strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); - pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + strncpy(pMsg, pCond->cond, pCond->len); + pMsg += pCond->len; } // compressed ts block @@ -979,6 +1000,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); } + memcpy(pMsg, pSql->sqlstr, sqlLen); + pMsg += sqlLen; + int32_t msgLen = (int32_t)(pMsg - pCmd->payload); tscDebug("%p msg built success, len:%d bytes", pSql, msgLen); @@ -1247,7 +1271,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pEpAddr->n); } - pCmd->payloadLen = sizeof(SShowMsg) + pShowMsg->payloadLen; + pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); return TSDB_CODE_SUCCESS; } @@ -1445,51 +1469,18 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + + SNewVgroupInfo vgroupInfo = {.vgId = -1}; + taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + assert(vgroupInfo.vgId > 0); - tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epSet); + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); return TSDB_CODE_SUCCESS; } -//int tscBuildCancelQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { -// SCancelQueryMsg *pCancelMsg = (SCancelQueryMsg*) pSql->cmd.payload; -// pCancelMsg->qhandle = htobe64(pSql->res.qhandle); -// -// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); -// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); -// -// if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { -// int32_t vgIndex = pTableMetaInfo->vgroupIndex; -// if (pTableMetaInfo->pVgroupTables == NULL) { -// SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; -// assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); -// -// pCancelMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); -// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex); -// } else { -// int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); -// assert(vgIndex >= 0 && vgIndex < numOfVgroups); -// -// SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); -// -// pCancelMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId); -// tscDebug("%p build cancel query msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex); -// } -// } else { -// STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; -// pCancelMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); -// tscDebug("%p build cancel query msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId); -// } -// -// pSql->cmd.payloadLen = sizeof(SCancelQueryMsg); -// pSql->cmd.msgType = TSDB_MSG_TYPE_CANCEL_QUERY; -// -// pCancelMsg->header.contLen = htonl(sizeof(SCancelQueryMsg)); -// return TSDB_CODE_SUCCESS; -//} - int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SAlterDbMsg); @@ -1551,7 +1542,7 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { if (code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -1580,7 +1571,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { int32_t code = pRes->code; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; } @@ -1599,7 +1590,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return code; @@ -1843,19 +1834,46 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pSchema++; } - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - // todo add one more function: taosAddDataIfNotExists(); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); - pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { + // check if super table hashmap or not + int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - if (pTableMetaInfo->pTableMeta == NULL) { - free(pTableMeta); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + // super tableMeta data alreay exists, create it according to tableMeta and add it to hash map + STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); + + uint32_t size = tscGetTableMetaSize(pSupTableMeta); + int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size); + assert(code == TSDB_CODE_SUCCESS); + + tfree(pSupTableMeta); + + CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); + taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), cMeta, sizeof(CChildTableMeta)); + tfree(cMeta); + } else { + uint32_t s = tscGetTableMetaSize(pTableMeta); + taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, s); + } + + // update the vgroupInfo if needed + if (pTableMeta->vgId > 0) { + int32_t vgId = pTableMeta->vgId; + assert(pTableMeta->tableType != TSDB_SUPER_TABLE); + + SNewVgroupInfo vgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); + + if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, &pMetaMsg->vgroup)) || + (vgroupInfo.inUse < 0)) { // vgroup info exists, compare with it + vgroupInfo = createNewVgroupInfo(&pMetaMsg->vgroup); + taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); + tscDebug("add new VgroupInfo, vgId:%d, total:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + } } tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name); @@ -1866,8 +1884,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { /** * multi table meta rsp pkg format: - * | STaosRsp | ieType | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 - * |...... 1B 1B 4B + * | STaosRsp | SMultiTableInfoMsg | SMeterMeta0 | SSchema0 | SMeterMeta1 | SSchema1 | SMeterMeta2 | SSchema2 + * |...... 1B 4B **/ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) { #if 0 @@ -2021,14 +2039,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { return pSql->res.code; } -/* - * current process do not use the cache at all - */ int tscProcessShowRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg; SShowRsp * pShow; SSchema * pSchema; - char key[20]; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -2053,20 +2067,10 @@ int tscProcessShowRsp(SSqlObj *pSql) { pSchema++; } - key[0] = pCmd->msgType + 'a'; - strcpy(key + 1, "showlist"); - - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false); - } + tfree(pTableMetaInfo->pTableMeta); + pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - size_t size = 0; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); - - pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } @@ -2089,12 +2093,9 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); - - tfree(pTableMeta); return 0; } -// TODO multithread problem static void createHBObj(STscObj* pObj) { if (pObj->hbrid != 0) { return; @@ -2176,51 +2177,34 @@ int tscProcessUseDbRsp(SSqlObj *pSql) { int tscProcessDropDbRsp(SSqlObj *pSql) { pSql->pTscObj->db[0] = 0; - taosCacheEmpty(tscMetaCache); + taosHashEmpty(tscTableMetaInfo); return 0; } int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } + //The cached tableMeta is expired in this case, so clean it in hash table + taosHashRemove(tscTableMetaInfo, pTableMetaInfo->name, strnlen(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN)); + tscDebug("%p remove table meta after drop table:%s, numOfRemain:%d", pSql, pTableMetaInfo->name, + (int32_t) taosHashGetSize(tscTableMetaInfo)); - /* - * 1. if a user drops one table, which is the only table in a vnode, remove operation will incur vnode to be removed. - * 2. Then, a user creates a new metric followed by a table with identical name of removed table but different schema, - * here the table will reside in a new vnode. - * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache - * instead. - */ - tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); assert(pTableMetaInfo->pTableMeta == NULL); - return 0; } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMeta == NULL) { /* not in cache, abort */ - return 0; - } - - tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name); - taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true); + char* name = pTableMetaInfo->name; + tscDebug("%p remove tableMeta in hashMap after alter-table: %s", pSql, name); - if (pTableMetaInfo->pTableMeta) { - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); + bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tfree(pTableMetaInfo->pTableMeta); - if (isSuperTable) { // if it is a super table, reset whole query cache - tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name); - taosCacheEmpty(tscMetaCache); - } + if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta + taosHashEmpty(tscTableMetaInfo); } return 0; @@ -2230,6 +2214,7 @@ int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { UNUSED(pSql); return 0; } + int tscProcessShowCreateRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 1); } @@ -2350,7 +2335,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated + code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated } return code; @@ -2358,21 +2343,29 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { assert(strlen(pTableMetaInfo->name) != 0); + tfree(pTableMetaInfo->pTableMeta); - // If this STableMetaInfo owns a table meta, release it first - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false); - } - - pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name)); - if (pTableMetaInfo->pTableMeta != NULL) { - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns, - tinfo.numOfTags, pTableMetaInfo->pTableMeta); + uint32_t size = tscGetTableMetaMaxSize(); + pTableMetaInfo->pTableMeta = calloc(1, size); + + pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; + int32_t len = (int32_t) strlen(pTableMetaInfo->name); + + taosHashGetClone(tscTableMetaInfo, pTableMetaInfo->name, len, NULL, pTableMetaInfo->pTableMeta, -1); + + // TODO resize the tableMeta + STableMeta* pMeta = pTableMetaInfo->pTableMeta; + if (pMeta->id.uid > 0) { + if (pMeta->tableType == TSDB_CHILD_TABLE) { + int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, pTableMetaInfo->name); + if (code != TSDB_CODE_SUCCESS) { + return getTableMetaFromMnode(pSql, pTableMetaInfo); + } + } return TSDB_CODE_SUCCESS; } - + return getTableMetaFromMnode(pSql, pTableMetaInfo); } @@ -2382,7 +2375,7 @@ int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create } /** - * retrieve table meta from mnode, and update the local table meta cache. + * retrieve table meta from mnode, and update the local table meta hashmap. * @param pSql sql object * @param tableIndex table index * @return status code @@ -2390,16 +2383,18 @@ int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); + const char* name = pTableMetaInfo->name; STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pTableMeta) { - tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta); + if (pTableMeta) { + tscDebug("%p update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64, pSql, name, + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); } - taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true); + // remove stored tableMeta info in hash table + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); return getTableMetaFromMnode(pSql, pTableMetaInfo); } @@ -2440,7 +2435,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); + STableMeta* pTableMeta = tscTableMetaClone(pMInfo->pTableMeta); tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d7dec2f35643dfb68e66e3d8197f3a735d5a700e..377cb24b1d5b1c92c702cd113c4feea185617911 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -115,9 +115,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa pObj->signature = pObj; pObj->pDnodeConn = pDnodeConn; - T_REF_INIT_VAL(pObj, 1); - tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); memcpy(pObj->pass, secretEncrypt, secretEncryptLen); @@ -172,11 +170,9 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (taos != NULL) { *taos = pObj; } - + pObj->rid = taosAddRef(tscRefId, pObj); registerSqlObj(pSql); - tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - pObj->rid = taosAddRef(tscRefId, pObj); return pSql; } @@ -288,34 +284,21 @@ void taos_close(TAOS *taos) { return; } - // make sure that the close connection can only be executed once. - pObj->signature = NULL; - taosTmrStopA(&(pObj->pTimer)); - - if (pObj->hbrid > 0) { + if (RID_VALID(pObj->hbrid)) { SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); if (pHb != NULL) { - if (pHb->rpcRid > 0) { // wait for rsp from dnode + if (RID_VALID(pHb->rpcRid)) { // wait for rsp from dnode rpcCancelRequest(pHb->rpcRid); pHb->rpcRid = -1; } tscDebug("%p HB is freed", pHb); - taos_free_result(pHb); taosReleaseRef(tscObjRef, pHb->self); + taos_free_result(pHb); } } - int32_t ref = T_REF_DEC(pObj); - assert(ref >= 0); - - if (ref > 0) { - tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn); - return; - } - tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); - taosRemoveRef(tscRefId, pObj->rid); } @@ -331,7 +314,7 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { tsem_post(&pSql->rspSem); } -TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES** res) { +TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, int64_t* res) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { terrno = TSDB_CODE_TSC_DISCONNECTED; @@ -357,7 +340,7 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen, TAOS_RES doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); if (res != NULL) { - *res = pSql; + atomic_store_64(res, pSql->self); } tsem_wait(&pSql->rspSem); @@ -368,7 +351,7 @@ TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { return taos_query_c(taos, sqlstr, (uint32_t)strlen(sqlstr), NULL); } -TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, TAOS_RES** res) { +TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res) { return taos_query_c(taos, sqlstr, (uint32_t) strlen(sqlstr), res); } @@ -726,7 +709,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) { pSubObj->rpcRid = -1; } - tscQueueAsyncRes(pSubObj); + tscAsyncResultOnError(pSubObj); taosReleaseRef(tscObjRef, pSubObj->self); } @@ -762,7 +745,7 @@ void taos_stop_query(TAOS_RES *res) { pSql->rpcRid = -1; } - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } @@ -926,7 +909,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) { // must before clean the sqlcmd object - tscResetSqlCmdObj(&pSql->cmd, false); + tscResetSqlCmdObj(&pSql->cmd); SSqlCmd *pCmd = &pSql->cmd; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 74b8e4d95878d8a784d4d82e28028e94bd32207b..c1ed9b0ba09f956207eeeccb2c7b8123538a2151 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -167,7 +167,9 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf retryDelay); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); + + char* name = pTableMetaInfo->name; + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); @@ -269,9 +271,8 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name, pStream->numOfRes); - // release the metric/meter meta information reference, so data in cache can be updated + tfree(pTableMetaInfo->pTableMeta); - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 52b74f7502db139f18d8230eefb36d41bae4079f..7f0b174ad364c783d03b3e7c7ee9bf8854c27cc7 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -69,14 +69,17 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { } void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { - if( sub == NULL) + if( sub == NULL) { return; + } + SSub* pSub = (SSub*)sub; SSubscriptionProgress target = {.uid = uid, .key = ts}; SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); if (p != NULL) { p->key = ts; + tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key); } } @@ -502,6 +505,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key; + tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); } if (pSub->pTimer == NULL) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index a328ae4d0485690d5dc392a1b9ad5e33a1b930f1..681291d0db7039af09147d86f6db1fa48ccb233c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -779,7 +779,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = numOfRows; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -796,7 +796,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -845,7 +845,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (code != TSDB_CODE_SUCCESS) { freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); taosArrayDestroy(s1); taosArrayDestroy(s2); @@ -916,7 +916,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pParentSql->res.code = numOfRows; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -930,7 +930,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1028,7 +1028,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pParentSql->res.code = numOfRows; tscError("%p retrieve failed, index:%d, code:%s", pSql, pSupporter->subqueryIndex, tstrerror(numOfRows)); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1155,7 +1155,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1233,7 +1233,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } return; @@ -1344,7 +1344,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pParentSql->res.code != TSDB_CODE_SUCCESS) { tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code); quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1357,7 +1357,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { pParentSql->res.code = code; quitAllSubquery(pParentSql, pSupporter); - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); return; } @@ -1403,7 +1403,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1612,7 +1612,7 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { _error: pRes->code = code; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { @@ -1666,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, &pFinalModel, nBufferSize); if (ret != 0) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); tfree(pMemoryBuf); return ret; } @@ -1680,7 +1680,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pFinalModel,pState->numOfSub); - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return ret; } @@ -1890,7 +1890,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); } else { // regular super table query if (pParentSql->res.code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } } @@ -1968,7 +1968,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p if (pParentSql->res.code == TSDB_CODE_SUCCESS) { (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - tscQueueAsyncRes(pParentSql); + tscAsyncResultOnError(pParentSql); } } @@ -2220,7 +2220,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) (*pParentObj->fp)(pParentObj->param, pParentObj, v); } else { if (!needRetryInsert(pParentObj, numOfSub)) { - tscQueueAsyncRes(pParentObj); + tscAsyncResultOnError(pParentObj); return; } @@ -2231,7 +2231,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) numOfFailed += 1; // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd, true); + tscFreeQueryInfo(&pSql->cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); @@ -2243,15 +2243,16 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) tscError("%p Async insertion completed, total inserted:%d rows, numOfFailed:%d, numOfTotal:%d", pParentObj, pParentObj->res.numOfRows, numOfFailed, numOfSub); - tscDebug("%p cleanup %d tableMeta in cache", pParentObj, pParentObj->cmd.numOfTables); + tscDebug("%p cleanup %d tableMeta in hashTable", pParentObj, pParentObj->cmd.numOfTables); for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) { - taosCacheRelease(tscMetaCache, (void**)&(pParentObj->cmd.pTableMetaList[i]), true); + char* name = pParentObj->cmd.pTableNameList[i]; + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } pParentObj->cmd.parseFinished = false; pParentObj->subState.numOfRemain = numOfFailed; - tscResetSqlCmdObj(&pParentObj->cmd, false); + tscResetSqlCmdObj(&pParentObj->cmd); // in case of insert, redo parsing the sql string and build new submit data block for two reasons: // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. @@ -2264,7 +2265,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) if (code != TSDB_CODE_SUCCESS) { pParentObj->res.code = code; - tscQueueAsyncRes(pParentObj); + tscAsyncResultOnError(pParentObj); return; } @@ -2288,7 +2289,7 @@ int32_t tscHandleInsertRetry(SSqlObj* pParent, SSqlObj* pSql) { int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return code; // here the pSql may have been released already. } @@ -2481,7 +2482,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SSqlRes* pRes = &pSql->res; if (pRes->code != TSDB_CODE_SUCCESS) { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -2496,7 +2497,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->tsrow == NULL || pRes->buffer == NULL || pRes->length == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); return; } @@ -2508,7 +2509,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { - tscQueueAsyncRes(pSql); + tscAsyncResultOnError(pSql); } } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 1eddeacc6514356d7a45874c33e9132443a83eca..dd0f248b85452342fb567357ba9346ab71754dfc 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -31,18 +31,21 @@ #include "tlocale.h" // global, not configurable -SCacheObj* tscMetaCache; -int tscObjRef = -1; -void * tscTmr; -void * tscQhandle; -void * tscCheckDiskUsageTmr; -int tsInsertHeadSize; -int tscRefId = -1; +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 -int tscNumOfThreads; +int32_t sentinel = TSC_VAR_NOT_RELEASE; +SHashObj *tscVgroupMap; // hash map to keep the global vgroup info +SHashObj *tscTableMetaInfo; // table meta info +int32_t tscObjRef = -1; +void *tscTmr; +void *tscQhandle; +int32_t tscRefId = -1; +int32_t tscNumOfObj = 0; // number of sqlObj in current process. + +static void *tscCheckDiskUsageTmr; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -//void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet); void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); @@ -115,7 +118,7 @@ void taos_init_imp(void) { int queueSize = tsMaxConnections*2; double factor = (tscEmbedded == 0)? 2.0:4.0; - tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); + int32_t tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); if (tscNumOfThreads < 2) { tscNumOfThreads = 2; } @@ -131,10 +134,11 @@ void taos_init_imp(void) { taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); } - int64_t refreshTime = 10; // 10 seconds by default - if (tscMetaCache == NULL) { - tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); - tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + if (tscTableMetaInfo == NULL) { + tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); + tscVgroupMap = taosHashInit(256, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + tscTableMetaInfo = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tscDebug("TableMeta:%p", tscTableMetaInfo); } tscRefId = taosOpenRef(200, tscCloseTscObj); @@ -152,30 +156,38 @@ void taos_init() { pthread_once(&tscinit, taos_init_imp); } void taos_cleanup(void) { tscDebug("start to cleanup client environment"); - void* m = tscMetaCache; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscMetaCache, m, 0) == m) { - taosCacheCleanup(m); + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; } - int refId = atomic_exchange_32(&tscObjRef, -1); - if (refId != -1) { - taosCloseRef(refId); - } + taosHashCleanup(tscTableMetaInfo); + tscTableMetaInfo = NULL; - m = tscQhandle; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscQhandle, m, 0) == m) { - taosCleanUpScheduler(m); - } + taosHashCleanup(tscVgroupMap); + tscVgroupMap = NULL; + + int32_t id = tscObjRef; + tscObjRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscRefId; + tscRefId = -1; + taosCloseRef(id); - taosCloseRef(tscRefId); taosCleanupKeywordsTable(); taosCloseLog(); - if (tscEmbedded == 0) rpcCleanup(); - m = tscTmr; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) { - taosTmrCleanUp(m); + if (tscEmbedded == 0) { + rpcCleanup(); } + + p = tscTmr; + tscTmr = NULL; + taosTmrCleanUp(p); } static int taos_options_imp(TSDB_OPTION option, const char *pStr) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 99cddc17d36212d8da256c215619e276eb14cf1e..5d818692ed90480c4b461c1fe8aba9d785dee694 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -18,7 +18,6 @@ #include "os.h" #include "qAst.h" #include "taosmsg.h" -#include "tcache.h" #include "tkey.h" #include "tmd5.h" #include "tscLocalMerge.h" @@ -31,7 +30,7 @@ #include "ttokendef.h" static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); -static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache); +static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo); SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { @@ -379,17 +378,16 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { +void tscFreeQueryInfo(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } for (int32_t i = 0; i < pCmd->numOfClause; ++i) { - char* addr = (char*)pCmd - offsetof(SSqlObj, cmd); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache); + clearAllTableMetaInfo(pQueryInfo); tfree(pQueryInfo); } @@ -397,7 +395,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { tfree(pCmd->pQueryInfo); } -void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { +void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; @@ -407,17 +405,17 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->autoCreated = 0; for(int32_t i = 0; i < pCmd->numOfTables; ++i) { - if (pCmd->pTableMetaList && pCmd->pTableMetaList[i]) { - taosCacheRelease(tscMetaCache, (void**)&(pCmd->pTableMetaList[i]), false); + if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { + tfree(pCmd->pTableNameList[i]); } } pCmd->numOfTables = 0; - tfree(pCmd->pTableMetaList); + tfree(pCmd->pTableNameList); pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd, removeFromCache); + tscFreeQueryInfo(pCmd); } void tscFreeSqlResult(SSqlObj* pSql) { @@ -458,35 +456,15 @@ void tscFreeRegisteredSqlObj(void *pSql) { SSqlObj* p = *(SSqlObj**)pSql; STscObj* pTscObj = p->pTscObj; - assert(p->self != 0); - tscFreeSqlObj(p); - - int32_t ref = T_REF_DEC(pTscObj); - assert(ref >= 0); - - tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref); - if (ref == 0) { - tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj); - taosRemoveRef(tscRefId, pTscObj->rid); - } -} + assert(RID_VALID(p->self)); -void tscFreeTableMetaHelper(void *pTableMeta) { - STableMeta* p = (STableMeta*) pTableMeta; + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfObj, 1); + int32_t total = atomic_sub_fetch_32(&tscNumOfObj, 1); - int32_t numOfEps = p->vgroupInfo.numOfEps; - assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA); - - for(int32_t i = 0; i < numOfEps; ++i) { - tfree(p->vgroupInfo.epAddr[i].fqdn); - } - - int32_t numOfEps1 = p->corVgroupInfo.numOfEps; - assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA); + tscDebug("%p free SqlObj, total in tscObj:%d, total:%d", pSql, num, total); + tscFreeSqlObj(p); + taosReleaseRef(tscRefId, pTscObj->rid); - for(int32_t i = 0; i < numOfEps1; ++i) { - tfree(p->corVgroupInfo.epAddr[i].fqdn); - } } void tscFreeSqlObj(SSqlObj* pSql) { @@ -516,7 +494,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->self = 0; tscFreeSqlResult(pSql); - tscResetSqlCmdObj(pCmd, false); + tscResetSqlCmdObj(pCmd); tfree(pCmd->tagData.data); pCmd->tagData.dataLen = 0; @@ -539,7 +517,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); + tfree(pDataBlock->pTableMeta); } tfree(pDataBlock); @@ -610,15 +588,15 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { - tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name)); + tstrncpy(pTableMetaInfo->name, pDataBlock->tableName, sizeof(pTableMetaInfo->name)); if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); + tfree(pTableMetaInfo->pTableMeta); } - pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta); + pTableMetaInfo->pTableMeta = tscTableMetaClone(pDataBlock->pTableMeta); } else { - assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0); + assert(strncmp(pTableMetaInfo->name, pDataBlock->tableName, tListLen(pDataBlock->tableName)) == 0); } /* @@ -681,14 +659,10 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff dataBuf->size = startOffset; dataBuf->tsSource = -1; - tstrncpy(dataBuf->tableId, name, sizeof(dataBuf->tableId)); + tstrncpy(dataBuf->tableName, name, sizeof(dataBuf->tableName)); - /* - * The table meta may be released since the table meta cache are completed clean by other thread - * due to operation such as drop database. So here we add the reference count directly instead of invoke - * taosGetDataFromCache, which may return NULL value. - */ - dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta); + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tscTableMetaClone(pTableMeta); assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; @@ -794,15 +768,15 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { return result; } -static void extractTableMeta(SSqlCmd* pCmd) { +static void extractTableNameList(SSqlCmd* pCmd) { pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList); - pCmd->pTableMetaList = calloc(pCmd->numOfTables, POINTER_BYTES); + pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES); STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL); int32_t i = 0; while(p1) { STableDataBlocks* pBlocks = *p1; - pCmd->pTableMetaList[i++] = taosCacheTransfer(tscMetaCache, (void**) &pBlocks->pTableMeta); + pCmd->pTableNameList[i++] = strndup(pBlocks->tableName, TSDB_TABLE_FNAME_LEN); p1 = taosHashIterate(pCmd->pTableBlockHashList, p1); } @@ -810,6 +784,7 @@ static void extractTableMeta(SSqlCmd* pCmd) { } int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { + const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); SSqlCmd* pCmd = &pSql->cmd; void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); @@ -824,7 +799,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { STableDataBlocks* dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - tsInsertHeadSize, 0, pOneTableBlock->tableId, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); taosHashCleanup(pVnodeDataBlockHashList); @@ -858,7 +833,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { tscSortRemoveDataBlockDupRows(pOneTableBlock); char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - tscDebug("%p tableId:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableId, + tscDebug("%p name:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableName, pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); @@ -888,7 +863,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { pOneTableBlock = *p; } - extractTableMeta(pCmd); + extractTableNameList(pCmd); // free the table data blocks; pCmd->pDataBlocks = pVnodeDataBlockList; @@ -909,6 +884,7 @@ void tscCloseTscObj(void *param) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; } + tfree(pObj->tscCorMgmtEpSet); pthread_mutex_destroy(&pObj->mutex); @@ -1537,6 +1513,7 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { } dest->tbnameCond.uid = src->tbnameCond.uid; + dest->tbnameCond.len = src->tbnameCond.len; memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); dest->relType = src->relType; @@ -1832,14 +1809,12 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { return pa; } -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { - tscDebug("%p unref %d tables in the tableMeta cache", address, pQueryInfo->numOfTables); - +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); - tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); + tscClearTableMetaInfo(pTableMetaInfo); free(pTableMetaInfo); } @@ -1893,14 +1868,12 @@ STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); } -void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { +void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo) { if (pTableMetaInfo == NULL) { return; } - if (pTableMetaInfo->pTableMeta != NULL) { - taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - } + tfree(pTableMetaInfo->pTableMeta); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); @@ -1917,10 +1890,12 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { } void registerSqlObj(SSqlObj* pSql) { - int32_t ref = T_REF_INC(pSql->pTscObj); - tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref); - + taosAcquireRef(tscRefId, pSql->pTscObj->rid); pSql->self = taosAddRef(tscObjRef, pSql); + + int32_t num = atomic_add_fetch_32(&pSql->pTscObj->numOfObj, 1); + int32_t total = atomic_add_fetch_32(&tscNumOfObj, 1); + tscDebug("%p new SqlObj from %p, total in tscObj:%d, total:%d", pSql, pSql->pTscObj, num, total); } SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) { @@ -1950,30 +1925,24 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return NULL; } - pNew->fp = fp; + pNew->fp = fp; pNew->fetchFp = fp; - pNew->param = param; + pNew->param = param; + pNew->sqlstr = NULL; pNew->maxRetry = TSDB_MAX_REPLICA; - pNew->sqlstr = strdup(pSql->sqlstr); - if (pNew->sqlstr == NULL) { - tscError("%p new subquery failed", pSql); - tscFreeSqlObj(pNew); - return NULL; - } - SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); - registerSqlObj(pNew); + return pNew; } -static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) { +static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t uid) { int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo); if (numOfOutput == 0) { return; @@ -2026,15 +1995,9 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, tableIndex); - pNew->pTscObj = pSql->pTscObj; + pNew->pTscObj = pSql->pTscObj; pNew->signature = pNew; - - pNew->sqlstr = strdup(pSql->sqlstr); - if (pNew->sqlstr == NULL) { - tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _error; - } + pNew->sqlstr = strdup(pSql->sqlstr); SSqlCmd* pnCmd = &pNew->cmd; memcpy(pnCmd, pCmd, sizeof(SSqlCmd)); @@ -2050,7 +2013,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->numOfTables = 0; pnCmd->parseFinished = 1; - pnCmd->pTableMetaList = NULL; + pnCmd->pTableNameList = NULL; pnCmd->pTableBlockHashList = NULL; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { @@ -2122,35 +2085,34 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void goto _error; } - doSetSqlExprAndResultFieldInfo(pQueryInfo, pNewQueryInfo, uid); + doSetSqlExprAndResultFieldInfo(pNewQueryInfo, uid); - pNew->fp = fp; + pNew->fp = fp; pNew->fetchFp = fp; - - pNew->param = param; + pNew->param = param; pNew->maxRetry = TSDB_MAX_REPLICA; char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { - STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup + STableMeta* pTableMeta = tscTableMetaClone(pTableMetaInfo->pTableMeta); assert(pTableMeta != NULL); pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, - pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); + pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); - + STableMeta* pPrevTableMeta = tscTableMetaClone(pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } + // this case cannot be happened if (pFinalInfo->pTableMeta == NULL) { - tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); + tscError("%p new subquery failed since no tableMeta, name:%s", pSql, name); if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); @@ -2523,7 +2485,7 @@ bool tscSetSqlOwner(SSqlObj* pSql) { SSqlRes* pRes = &pSql->res; // set the sql object owner - uint64_t threadId = taosGetPthreadId(); + uint64_t threadId = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; @@ -2577,6 +2539,7 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { tfree(pVgroupInfo->epAddr[j].fqdn); } + for(int32_t j = pVgroupInfo->numOfEps; j < TSDB_MAX_REPLICA; j++) { assert( pVgroupInfo->epAddr[j].fqdn == NULL ); } @@ -2629,4 +2592,88 @@ int32_t copyTagData(STagData* dst, const STagData* src) { } return 0; -} \ No newline at end of file +} + +STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { + assert(pChild != NULL); + int32_t total = pChild->numOfColumns + pChild->numOfTags; + + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); + pTableMeta->tableType = TSDB_SUPER_TABLE; + pTableMeta->tableInfo.numOfTags = pChild->numOfTags; + pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; + pTableMeta->tableInfo.precision = pChild->precision; + + pTableMeta->id.tid = 0; + pTableMeta->id.uid = pChild->suid; + pTableMeta->tversion = pChild->tversion; + pTableMeta->sversion = pChild->sversion; + + memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); + + int32_t num = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < num; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + int32_t totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + return sizeof(STableMeta) + totalCols * sizeof(SSchema); +} + +CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + + CChildTableMeta* cMeta = calloc(1, sizeof(CChildTableMeta)); + cMeta->tableType = TSDB_CHILD_TABLE; + cMeta->vgId = pTableMeta->vgId; + cMeta->id = pTableMeta->id; + tstrncpy(cMeta->sTableName, pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); + + return cMeta; +} + +int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name) { + assert(pChild != NULL); + + uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* p = calloc(1, size); + + taosHashGetClone(tscTableMetaInfo, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p, -1); + if (p->id.uid > 0) { // tableMeta exists, build child table meta and return + pChild->sversion = p->sversion; + pChild->tversion = p->tversion; + + memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableInfo)); + int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; + + memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); + + tfree(p); + return TSDB_CODE_SUCCESS; + } else { // super table has been removed, current tableMeta is also expired. remove it here + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + + tfree(p); + return -1; + } +} + +uint32_t tscGetTableMetaMaxSize() { + return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); +} + +STableMeta* tscTableMetaClone(STableMeta* pTableMeta) { + assert(pTableMeta != NULL); + uint32_t size = tscGetTableMetaSize(pTableMeta); + STableMeta* p = calloc(1, size); + memcpy(p, pTableMeta, size); + return p; +} + + diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index bf1f22a4ee686b6b20815283c43d8c47ca933da2..f1fc5ca8087ddfca0f03c84d2e93c064aad761ce 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -32,8 +32,8 @@ extern uint16_t tsSyncPort; extern uint16_t tsArbitratorPort; extern int32_t tsStatusInterval; extern int32_t tsNumOfMnodes; -extern int32_t tsEnableVnodeBak; -extern int32_t tsEnableTelemetryReporting; +extern int8_t tsEnableVnodeBak; +extern int8_t tsEnableTelemetryReporting; extern char tsEmail[]; extern char tsArbitrator[]; @@ -51,7 +51,7 @@ extern int8_t tsDaylight; extern char tsTimezone[]; extern char tsLocale[]; extern char tsCharset[]; // default encode string -extern int32_t tsEnableCoreFile; +extern int8_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; extern char tsTempDir[]; @@ -59,12 +59,12 @@ extern char tsTempDir[]; extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked -extern int32_t tsKeepOriginalColumnName; +extern int8_t tsKeepOriginalColumnName; // client extern int32_t tsTableMetaKeepTimer; extern int32_t tsMaxSQLStringLen; -extern int32_t tsTscEnableRecordSql; +extern int8_t tsTscEnableRecordSql; extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; @@ -93,48 +93,51 @@ extern int16_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; -extern int32_t tsUpdate; +extern int8_t tsUpdate; +extern int8_t tsCacheLastRow; // balance -extern int32_t tsEnableBalance; -extern int32_t tsAlternativeRole; +extern int8_t tsEnableBalance; +extern int8_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; extern int32_t tsMnodeEqualVnodeNum; -extern int32_t tsFlowCtrl; +extern int8_t tsEnableFlowCtrl; +extern int8_t tsEnableSlaveQuery; +extern int8_t tsEnableAdjustMaster; // restful -extern int32_t tsEnableHttpModule; +extern int8_t tsEnableHttpModule; extern int32_t tsRestRowLimit; extern uint16_t tsHttpPort; extern int32_t tsHttpCacheSessions; extern int32_t tsHttpSessionExpire; extern int32_t tsHttpMaxThreads; -extern int32_t tsHttpEnableCompress; -extern int32_t tsHttpEnableRecordSql; -extern int32_t tsTelegrafUseFieldNum; +extern int8_t tsHttpEnableCompress; +extern int8_t tsHttpEnableRecordSql; +extern int8_t tsTelegrafUseFieldNum; // mqtt -extern int32_t tsEnableMqttModule; -extern char tsMqttHostName[]; -extern char tsMqttPort[]; -extern char tsMqttUser[]; -extern char tsMqttPass[]; -extern char tsMqttClientId[]; -extern char tsMqttTopic[]; +extern int8_t tsEnableMqttModule; +extern char tsMqttHostName[]; +extern char tsMqttPort[]; +extern char tsMqttUser[]; +extern char tsMqttPass[]; +extern char tsMqttClientId[]; +extern char tsMqttTopic[]; // monitor -extern int32_t tsEnableMonitorModule; +extern int8_t tsEnableMonitorModule; extern char tsMonitorDbName[]; extern char tsInternalPass[]; extern int32_t tsMonitorInterval; // stream -extern int32_t tsEnableStream; +extern int8_t tsEnableStream; // internal -extern int32_t tsPrintAuth; -extern uint32_t tscEmbedded; +extern int8_t tsPrintAuth; +extern int8_t tscEmbedded; extern char configDir[]; extern char tsVnodeDir[]; extern char tsDnodeDir[]; @@ -161,7 +164,7 @@ extern float tsMinimalLogDirGB; extern float tsReservedTmpDirectorySpace; extern float tsMinimalDataDirGB; extern int32_t tsTotalMemoryMB; -extern int32_t tsVersion; +extern uint32_t tsVersion; // build info extern char version[]; @@ -171,13 +174,13 @@ extern char gitinfoOfInternal[]; extern char buildinfo[]; // log -extern int32_t tsAsyncLog; +extern int8_t tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; extern int32_t vDebugFlag; extern int32_t mDebugFlag; -extern uint32_t cDebugFlag; +extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; extern int32_t sdbDebugFlag; @@ -187,7 +190,7 @@ extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t odbcDebugFlag; -extern uint32_t qDebugFlag; +extern int32_t qDebugFlag; extern int32_t wDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; diff --git a/src/common/inc/tulog.h b/src/common/inc/tulog.h index 74f572619bed27e7cdc0ba381d74213b23844645..566da40a10e078b9789e2e1b76a8d82fe89aef46 100644 --- a/src/common/inc/tulog.h +++ b/src/common/inc/tulog.h @@ -23,7 +23,7 @@ extern "C" { #include "tlog.h" extern int32_t uDebugFlag; -extern uint32_t tscEmbedded; +extern int8_t tscEmbedded; #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 279a2fef048b0de5d1aeee07a0b0d404f43cb9b5..8fa17f87511ca02759b5700dd59735a526da7fe9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -39,8 +39,8 @@ uint16_t tsSyncPort = 6040; uint16_t tsArbitratorPort = 6042; int32_t tsStatusInterval = 1; // second int32_t tsNumOfMnodes = 3; -int32_t tsEnableVnodeBak = 1; -int32_t tsEnableTelemetryReporting = 1; +int8_t tsEnableVnodeBak = 1; +int8_t tsEnableTelemetryReporting = 1; char tsEmail[TSDB_FQDN_LEN] = {0}; // common @@ -56,7 +56,7 @@ int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string -int32_t tsEnableCoreFile = 0; +int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; @@ -73,7 +73,7 @@ int32_t tsCompressMsgSize = -1; // client int32_t tsTableMetaKeepTimer = 7200; // second int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; -int32_t tsTscEnableRecordSql = 0; +int8_t tsTscEnableRecordSql = 0; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -110,7 +110,7 @@ int32_t tsQueryBufferSize = -1; int32_t tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name -int32_t tsKeepOriginalColumnName = 0; +int8_t tsKeepOriginalColumnName = 0; // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; @@ -126,33 +126,36 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; +int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; +int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; // balance -int32_t tsEnableBalance = 1; -int32_t tsAlternativeRole = 0; -int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400*100; // seconds 10days +int8_t tsEnableBalance = 1; +int8_t tsAlternativeRole = 0; +int32_t tsBalanceInterval = 300; // seconds +int32_t tsOfflineThreshold = 86400 * 100; // seconds 10days int32_t tsMnodeEqualVnodeNum = 4; -int32_t tsFlowCtrl = 1; +int8_t tsEnableFlowCtrl = 1; +int8_t tsEnableSlaveQuery = 1; +int8_t tsEnableAdjustMaster = 1; // restful -int32_t tsEnableHttpModule = 1; +int8_t tsEnableHttpModule = 1; int32_t tsRestRowLimit = 10240; uint16_t tsHttpPort = 6041; // only tcp, range tcp[6041] int32_t tsHttpCacheSessions = 1000; int32_t tsHttpSessionExpire = 36000; int32_t tsHttpMaxThreads = 2; -int32_t tsHttpEnableCompress = 1; -int32_t tsHttpEnableRecordSql = 0; -int32_t tsTelegrafUseFieldNum = 0; +int8_t tsHttpEnableCompress = 1; +int8_t tsHttpEnableRecordSql = 0; +int8_t tsTelegrafUseFieldNum = 0; // mqtt -int32_t tsEnableMqttModule = 0; // not finished yet, not started it by default +int8_t tsEnableMqttModule = 0; // not finished yet, not started it by default char tsMqttHostName[TSDB_MQTT_HOSTNAME_LEN] = "test.mosquitto.org"; char tsMqttPort[TSDB_MQTT_PORT_LEN] = "1883"; char tsMqttUser[TSDB_MQTT_USER_LEN] = {0}; @@ -161,24 +164,24 @@ char tsMqttClientId[TSDB_MQTT_CLIENT_ID_LEN] = "TDengineMqttSubscriber"; char tsMqttTopic[TSDB_MQTT_TOPIC_LEN] = "/test"; // # // monitor -int32_t tsEnableMonitorModule = 1; +int8_t tsEnableMonitorModule = 1; char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsInternalPass[] = "secretkey"; int32_t tsMonitorInterval = 30; // seconds // stream -int32_t tsEnableStream = 1; +int8_t tsEnableStream = 1; // internal -int32_t tsPrintAuth = 0; -uint32_t tscEmbedded = 0; -char configDir[TSDB_FILENAME_LEN] = {0}; -char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsDataDir[TSDB_FILENAME_LEN] = {0}; -char tsScriptDir[TSDB_FILENAME_LEN] = {0}; -char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; +int8_t tsPrintAuth = 0; +int8_t tscEmbedded = 0; +char configDir[TSDB_FILENAME_LEN] = {0}; +char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; +char tsDataDir[TSDB_FILENAME_LEN] = {0}; +char tsScriptDir[TSDB_FILENAME_LEN] = {0}; +char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; /* * minimum scale for whole system, millisecond by default @@ -198,10 +201,10 @@ float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; float tsAvailTmpDirectorySpace = 0; float tsAvailDataDirGB = 0; -float tsReservedTmpDirectorySpace = 0.1f; -float tsMinimalDataDirGB = 0.5f; +float tsReservedTmpDirectorySpace = 1.0f; +float tsMinimalDataDirGB = 1.0f; int32_t tsTotalMemoryMB = 0; -int32_t tsVersion = 0; +uint32_t tsVersion = 0; // log int32_t tsNumOfLogLines = 10000000; @@ -209,13 +212,13 @@ int32_t mDebugFlag = 131; int32_t sdbDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; -uint32_t cDebugFlag = 131; +int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; int32_t httpDebugFlag = 131; int32_t mqttDebugFlag = 131; int32_t monDebugFlag = 131; -uint32_t qDebugFlag = 131; +int32_t qDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; int32_t debugFlag = 0; @@ -275,12 +278,16 @@ bool taosCfgDynamicOptions(char *msg) { for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (cfg->valType != TAOS_CFG_VTYPE_INT32) continue; + if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; - *((int32_t *)cfg->ptr) = vint; + if (cfg->valType != TAOS_CFG_VTYPE_INT32) { + *((int32_t *)cfg->ptr) = vint; + } else { + *((int8_t *)cfg->ptr) = (int8_t)vint; + } if (strncasecmp(cfg->option, "monitor", olen) == 0) { if (1 == vint) { @@ -468,7 +475,7 @@ static void doInitGlobalConfig(void) { cfg.option = "vnodeBak"; cfg.ptr = &tsEnableVnodeBak; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -478,7 +485,7 @@ static void doInitGlobalConfig(void) { cfg.option = "telemetryReporting"; cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -488,7 +495,7 @@ static void doInitGlobalConfig(void) { cfg.option = "balance"; cfg.ptr = &tsEnableBalance; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -509,7 +516,7 @@ static void doInitGlobalConfig(void) { // 0-any; 1-mnode; 2-vnode cfg.option = "role"; cfg.ptr = &tsAlternativeRole; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 2; @@ -542,7 +549,7 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsOfflineThreshold; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 5; + cfg.minValue = 3; cfg.maxValue = 7200000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; @@ -811,7 +818,7 @@ static void doInitGlobalConfig(void) { cfg.option = "update"; cfg.ptr = &tsUpdate; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_DB_UPDATE; cfg.maxValue = TSDB_MAX_DB_UPDATE; @@ -901,7 +908,7 @@ static void doInitGlobalConfig(void) { cfg.option = "keepColumnName"; cfg.ptr = &tsKeepOriginalColumnName; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 1; @@ -1004,8 +1011,28 @@ static void doInitGlobalConfig(void) { // module configs cfg.option = "flowctrl"; - cfg.ptr = &tsFlowCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.ptr = &tsEnableFlowCtrl; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "slaveQuery"; + cfg.ptr = &tsEnableSlaveQuery; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "adjustMaster"; + cfg.ptr = &tsEnableAdjustMaster; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1015,7 +1042,7 @@ static void doInitGlobalConfig(void) { cfg.option = "http"; cfg.ptr = &tsEnableHttpModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1025,7 +1052,7 @@ static void doInitGlobalConfig(void) { cfg.option = "mqtt"; cfg.ptr = &tsEnableMqttModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1035,7 +1062,7 @@ static void doInitGlobalConfig(void) { cfg.option = "monitor"; cfg.ptr = &tsEnableMonitorModule; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1045,7 +1072,7 @@ static void doInitGlobalConfig(void) { cfg.option = "stream"; cfg.ptr = &tsEnableStream; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1055,7 +1082,7 @@ static void doInitGlobalConfig(void) { cfg.option = "httpEnableRecordSql"; cfg.ptr = &tsHttpEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1065,7 +1092,7 @@ static void doInitGlobalConfig(void) { cfg.option = "telegrafUseFieldNum"; cfg.ptr = &tsTelegrafUseFieldNum; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 0; cfg.maxValue = 1; @@ -1116,7 +1143,7 @@ static void doInitGlobalConfig(void) { cfg.option = "asyncLog"; cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 1; @@ -1317,7 +1344,7 @@ static void doInitGlobalConfig(void) { cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1327,7 +1354,7 @@ static void doInitGlobalConfig(void) { cfg.option = "enableCoreFile"; cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 1; @@ -1443,17 +1470,28 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } + if (tsMaxTablePerVnode < tsMinTablePerVnode) { + uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", + tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode); + tsMaxTablePerVnode = tsMinTablePerVnode; + } + // todo refactor tsVersion = 0; - for (int i = 0; i < 10; i++) { + for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { if (version[i] >= '0' && version[i] <= '9') { - tsVersion = tsVersion * 10 + (version[i] - '0'); + ver = ver * 10 + (version[i] - '0'); + } else if (version[i] == '.') { + tsVersion |= ver & 0xFF; + tsVersion <<= 8; + + ver = 0; } else if (version[i] == 0) { + tsVersion |= ver & 0xFF; + break; } } - - tsVersion = 10 * tsVersion; tsDnodeShellPort = tsServerPort + TSDB_PORT_DNODESHELL; // udp[6035-6039] tcp[6035] tsDnodeDnodePort = tsServerPort + TSDB_PORT_DNODEDNODE; // udp/tcp diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 61f3f0c1d6f1f85c98918cea1e330654fff9dcf8..9865fc7127b39debc965f6be9bad6e8485169416 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -56,6 +56,12 @@ test + + mysql + mysql-connector-java + 5.1.47 + + org.apache.httpcomponents @@ -80,6 +86,7 @@ + diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index 8bd974e3c2f9ef71a16fa737b381421a7d5ff915..5260b780bd510edad6ef8ea9a481fa334cca50f6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -17,7 +17,7 @@ public class RestfulConnection implements Connection { private final String host; private final int port; private final Properties props; - private final String database; + private volatile String database; private final String url; /******************************************************/ private boolean isClosed; @@ -139,7 +139,9 @@ public class RestfulConnection implements Connection { public void setCatalog(String catalog) throws SQLException { if (isClosed()) throw new SQLException(CONNECTION_IS_CLOSED); - //nothing to do + synchronized (RestfulConnection.class) { + this.database = catalog; + } } @Override @@ -409,7 +411,9 @@ public class RestfulConnection implements Connection { public void setSchema(String schema) throws SQLException { if (isClosed()) throw new SQLException(CONNECTION_IS_CLOSED); - //nothing to do + synchronized (RestfulConnection.class) { + this.database = schema; + } } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 690b8a17e672700f9e1aaf9effe165f987b86b54..8b2276fbb055cd18c86173f387fb7c93e296fe43 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -109,7 +109,7 @@ public class RestfulStatement implements Statement { throw new SQLException("Database not specified or available"); final String url = "http://" + conn.getHost().trim() + ":" + conn.getPort() + "/rest/sql"; - HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); +// HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); String result = HttpClientPoolUtil.execute(url, sql); JSONObject jsonObject = JSON.parseObject(result); if (jsonObject.getString("status").equals("error")) { @@ -215,6 +215,7 @@ public class RestfulStatement implements Statement { //如果执行了use操作应该将当前Statement的catalog设置为新的database if (SqlSyntaxValidator.isUseSql(sql)) { this.database = sql.trim().replace("use", "").trim(); + this.conn.setCatalog(this.database); } if (this.database == null) throw new SQLException("Database not specified or available"); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java index 7bf8efffc14bc7f3098f77788fe4197a598c3e35..23e8796980ac419635ef6953deff1033d158962e 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java @@ -96,6 +96,7 @@ public class HttpClientPoolUtil { initPools(); } method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME, DEFAULT_CONTENT_TYPE, 0); + method.setHeader("Authorization", "Basic cm9vdDp0YW9zZGF0YQ=="); method.setHeader("Content-Type", "text/plain"); method.setEntity(new StringEntity(data, Charset.forName("UTF-8"))); HttpContext context = HttpClientContext.create(); diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index 656741ea16fd607212711fb694e3e20e6007b2ac..995babdb2b7827267d6a4b68cd9d380671823543 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -144,18 +144,9 @@ function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, micro=false) { data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset); let res = []; - let currOffset = 0; - // every 4 bytes, a character is encoded; - while (currOffset < data.length) { - let dataEntry = data.slice(currOffset, currOffset + nbytes); //one entry in a row under a column; - if (dataEntry.readInt64LE(0) == FieldTypes.C_NCHAR_NULL) { - res.push(null); - } - else { - res.push(dataEntry.toString("utf16le").replace(/\u0000/g, "")); - } - currOffset += nbytes; - } + let dataEntry = data.slice(0, nbytes); //one entry in a row under a column; + //TODO: should use the correct character encoding + res.push(dataEntry.toString("utf-8")); return res; } @@ -349,11 +340,13 @@ CTaosInterface.prototype.useResult = function useResult(result) { return fields; } CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { - let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data - let num_of_rows = this.libtaos.taos_fetch_block(result, pblock) - if (num_of_rows == 0) { + //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data + let pblock = this.libtaos.taos_fetch_row(result); + let num_of_rows = 1; + if (ref.isNull(pblock) == true) { return {block:null, num_of_rows:0}; } + var fieldL = this.libtaos.taos_fetch_lengths(result); let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO); @@ -361,26 +354,28 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) { var fieldlens = []; if (ref.isNull(fieldL) == false) { - for (let i = 0; i < fields.length; i ++) { - let plen = ref.reinterpret(fieldL, 4, i*4); + let plen = ref.reinterpret(fieldL, 4, i*4); let len = plen.readInt32LE(0); - fieldlens.push(len); + fieldlens.push(len); } } let blocks = new Array(fields.length); blocks.fill(null); - num_of_rows = Math.abs(num_of_rows); + //num_of_rows = Math.abs(num_of_rows); let offset = 0; - pblock = pblock.deref(); for (let i = 0; i < fields.length; i++) { pdata = ref.reinterpret(pblock,8,i*8); - pdata = ref.ref(pdata.readPointer()); - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + pdata = ref.ref(pdata.readPointer()); + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + blocks[i] = convertFunctions[fields[i]['type']](pdata, 1, fieldlens[i], offset, isMicro); + } } return {blocks: blocks, num_of_rows:Math.abs(num_of_rows)} } @@ -446,14 +441,18 @@ CTaosInterface.prototype.fetch_rows_a = function fetch_rows_a(result, callback, } if (numOfRows2 > 0){ for (let i = 0; i < fields.length; i++) { - if (!convertFunctions[fields[i]['type']] ) { - throw new errors.DatabaseError("Invalid data type returned from database"); - } - let prow = ref.reinterpret(row,8,i*8); - prow = prow.readPointer(); - prow = ref.ref(prow); - blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); - //offset += fields[i]['bytes'] * numOfRows2; + if(ref.isNull(pdata.readPointer())){ + blocks[i] = new Array(); + }else{ + if (!convertFunctions[fields[i]['type']] ) { + throw new errors.DatabaseError("Invalid data type returned from database"); + } + let prow = ref.reinterpret(row,8,i*8); + prow = prow.readPointer(); + prow = ref.ref(prow); + blocks[i] = convertFunctions[fields[i]['type']](prow, 1, fieldlens[i], offset, isMicro); + //offset += fields[i]['bytes'] * numOfRows2; + } } } callback(param2, result2, numOfRows2, blocks); diff --git a/src/connector/nodejs/nodetaos/taosresult.js b/src/connector/nodejs/nodetaos/taosresult.js index fd82f4e236508eff46c54f6f156b592a43319c20..4138ebbec6e1b792691d17a25b7c18d35b6a922a 100644 --- a/src/connector/nodejs/nodetaos/taosresult.js +++ b/src/connector/nodejs/nodetaos/taosresult.js @@ -25,6 +25,7 @@ function TaosResult(data, fields) { * @function pretty * @since 1.0.6 */ + TaosResult.prototype.pretty = function pretty() { let fieldsStr = ""; let sizing = []; @@ -46,8 +47,7 @@ TaosResult.prototype.pretty = function pretty() { row.data.forEach((entry, i) => { if (this.fields[i]._field.type == 9) { entry = entry.toTaosString(); - } - else { + } else { entry = entry == null ? 'null' : entry.toString(); } rowStr += entry diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index 3f0600a09ca73c3fadbed1b7e65c51b30004edf5..2d5cf45e1db9a65ce86d998afea6cd689bf63317 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,6 +1,6 @@ { "name": "td2.0-connector", - "version": "2.0.1", + "version": "2.0.4", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", "scripts": { diff --git a/src/connector/nodejs/test/test.js b/src/connector/nodejs/test/test.js index 73dac8b26ce07ff159ce3de535776e2495d1ebde..bf4bb2c54188d3eb0f9c7fb5306912effc7b0760 100644 --- a/src/connector/nodejs/test/test.js +++ b/src/connector/nodejs/test/test.js @@ -48,6 +48,7 @@ for (let i = 0; i < 10000; i++) { // Select console.log('select * from td_connector_test.all_types limit 3 offset 100;'); c1.execute('select * from td_connector_test.all_types limit 2 offset 100;'); + var d = c1.fetchall(); console.log(c1.fields); console.log(d); @@ -77,13 +78,33 @@ c1.query('select stddev(_double), stddev(_bigint), stddev(_float) from all_types }) // Binding arguments, and then using promise -var q = c1.query('select * from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) +var q = c1.query('select _nchar from td_connector_test.all_types where ts >= ? and _int > ? limit 100 offset 40;').bind(new Date(1231), 100) console.log(q.query); q.execute().then(function(r) { r.pretty(); }); +// test query null value +c1.execute("create table if not exists td_connector_test.weather(ts timestamp, temperature float, humidity int) tags(location nchar(64))"); +c1.execute("insert into t1 using weather tags('北京') values(now, 11.11, 11)"); +c1.execute("insert into t1(ts, temperature) values(now, 22.22)"); +c1.execute("insert into t1(ts, humidity) values(now, 33)"); +c1.query('select * from test.t1', true).then(function (result) { + result.pretty(); +}); + +var q = c1.query('select * from td_connector_test.weather'); +console.log(q.query); +q.execute().then(function(r) { + r.pretty(); +}); + +function sleep(sleepTime) { + for(var start = +new Date; +new Date - start <= sleepTime; ) { } +} + +sleep(10000); // Raw Async Testing (Callbacks, not promises) function cb2(param, result, rowCount, rd) { @@ -129,16 +150,21 @@ setTimeout(function(){ c1.fetchall_a(thisRes, cb4, param); },100); + // Async through promises var aq = c1.query('select count(*) from td_connector_test.all_types;',false); aq.execute_a().then(function(data) { data.pretty(); }); -c1.query('describe td_connector_test.stabletest;').execute_a().then(r=> r.pretty()); + +c1.query('describe td_connector_test.stabletest').execute_a().then(function(r){ + r.pretty() +}); + setTimeout(function(){ c1.query('drop database td_connector_test;'); },200); + setTimeout(function(){ conn.close(); },2000); - diff --git a/src/dnode/inc/dnodeCfg.h b/src/dnode/inc/dnodeCfg.h index d74303f3252e1fa54e174876bab5758be541955e..896b3f574c2e0d02a0d62048a411fa484d16130b 100644 --- a/src/dnode/inc/dnodeCfg.h +++ b/src/dnode/inc/dnodeCfg.h @@ -25,6 +25,7 @@ int32_t dnodeInitCfg(); void dnodeCleanupCfg(); void dnodeUpdateCfg(SDnodeCfg *cfg); int32_t dnodeGetDnodeId(); +void dnodeGetClusterId(char *clusterId); void dnodeGetCfg(int32_t *dnodeId, char *clusterId); #ifdef __cplusplus diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c index 89249d773b7c67b8939a7a70f65882f89f8f5190..f495dbe285c196f3e32d8a0ace8736c8f44d668d 100644 --- a/src/dnode/src/dnodeCfg.c +++ b/src/dnode/src/dnodeCfg.c @@ -51,6 +51,12 @@ int32_t dnodeGetDnodeId() { return dnodeId; } +void dnodeGetClusterId(char *clusterId) { + pthread_mutex_lock(&tsCfgMutex); + tstrncpy(clusterId, tsCfg.clusterId, TSDB_CLUSTER_ID_LEN); + pthread_mutex_unlock(&tsCfgMutex); +} + void dnodeGetCfg(int32_t *dnodeId, char *clusterId) { pthread_mutex_lock(&tsCfgMutex); *dnodeId = tsCfg.dnodeId; diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index 7c385a889d10fcedd819f6eb10cbdaaf4910739f..dc89487f8b4c5ef2d2434e815d2ba1d8a0311515 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -311,6 +311,14 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); + if (strcmp(epSet.fqdn[i], tsLocalFqdn) == 0) { + if ((epSet.port[i] == tsServerPort + TSDB_PORT_DNODEDNODE && !forShell) || + (epSet.port[i] == tsServerPort && forShell)) { + epSet.inUse = (i + 1) % epSet.numOfEps; + dDebug("mnode index:%d %s:%d set inUse to %d", i, epSet.fqdn[i], epSet.port[i], epSet.inUse); + } + } + epSet.port[i] = htons(epSet.port[i]); } diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 0fc6400d99f8dbcbc57caa2ed1265e36e18f1a93..9027c346f57c20d91d943c0df8d061c931240101 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -16,9 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tqueue.h" -#include "twal.h" #include "mnode.h" -#include "dnodeVMgmt.h" #include "dnodeMInfos.h" #include "dnodeMRead.h" diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 414b66653d123b785643cfdc96b429edbe8d58ad..8c9e22ef4b19133863f32140fb6ddae292caf5a9 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -18,7 +18,6 @@ #include "ttimer.h" #include "tqueue.h" #include "mnode.h" -#include "dnodeVMgmt.h" #include "dnodeMInfos.h" #include "dnodeMWrite.h" @@ -185,7 +184,19 @@ void dnodeReprocessMWriteMsg(void *pMsg) { dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); - dnodeSendRedirectMsg(pMsg, true); + if (pWrite->pBatchMasterMsg) { + ++pWrite->pBatchMasterMsg->received; + if (pWrite->pBatchMasterMsg->successed + pWrite->pBatchMasterMsg->received + >= pWrite->pBatchMasterMsg->expected) { + dnodeSendRedirectMsg(&pWrite->pBatchMasterMsg->rpcMsg, true); + dnodeFreeMWriteMsg(pWrite->pBatchMasterMsg); + } + + mnodeDestroySubMsg(pWrite); + + return; + } + dnodeSendRedirectMsg(&pWrite->rpcMsg, true); dnodeFreeMWriteMsg(pWrite); } else { dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index b5c4997337edb7c7ffb4d7dc1c59707e6bdc4955..517a9e9bc83fb454bb4e2c43e31cbd8c90ad91cd 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -113,6 +113,7 @@ static void dnodeCleanupTmr() { int32_t dnodeInitSystem() { dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); tscEmbedded = 1; + taosIgnSIGPIPE(); taosBlockSIGPIPE(); taosResolveCRC(); taosInitGlobalCfg(); @@ -120,7 +121,6 @@ int32_t dnodeInitSystem() { taosSetCoreDump(); taosInitNotes(); dnodeInitTmr(); - signal(SIGPIPE, SIG_IGN); if (dnodeCreateDir(tsLogDir) < 0) { printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index de0c360c88e8197aea47beee656c7ddb77a26d8e..5ee10abc30311792bcd68e579567eea74fa93821 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -60,7 +60,7 @@ int32_t dnodeInitServer() { rpcInit.label = "DND-S"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessReqMsgFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES; + rpcInit.sessions = TSDB_MAX_VNODES << 4; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; @@ -123,7 +123,7 @@ int32_t dnodeInitClient() { rpcInit.label = "DND-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = dnodeProcessRspFromDnode; - rpcInit.sessions = TSDB_MAX_VNODES; + rpcInit.sessions = TSDB_MAX_VNODES << 4; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = "t"; diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 85f0137d896ff65440117dcc6fc6e2a3c1a28038..ff9598ecc563ac48bac051027385ab188e94ce75 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -195,7 +195,7 @@ static void addRuntimeInfo(SBufferWriter* bw) { static void sendTelemetryReport() { char buf[128]; - uint32_t ip = taosGetIpFromFqdn(TELEMETRY_SERVER); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; @@ -308,4 +308,4 @@ void dnodeCleanupTelemetry() { pthread_join(tsTelemetryThread, NULL); tsem_destroy(&tsExitSem); } -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c index 4753ebc40095ec8a7c6579781b2b8b8d83854432..bc24d1bf623ec014dd4a4ad35442218549aaf335 100644 --- a/src/dnode/src/dnodeVMgmt.c +++ b/src/dnode/src/dnodeVMgmt.c @@ -143,7 +143,7 @@ static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) { pCreate->cfg.fsyncPeriod = htonl(pCreate->cfg.fsyncPeriod); pCreate->cfg.commitTime = htonl(pCreate->cfg.commitTime); - for (int32_t j = 0; j < pCreate->cfg.replications; ++j) { + for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) { pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId); } @@ -217,4 +217,4 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { dnodeStartMnode(&pCfg->mnodes); return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 3f31e4937052d7a505031b4c8083ae1acd00bf6c..ea738661ce2813e13468ad91b4dc1d54775db21f 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -54,6 +54,7 @@ void dnodeCleanupVRead() { void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { int32_t queuedMsgNum = 0; int32_t leftLen = pMsg->contLen; + int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID; char * pCont = pMsg->pCont; while (leftLen > 0) { @@ -64,7 +65,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { assert(pHead->contLen > 0); void *pVnode = vnodeAcquire(pHead->vgId); if (pVnode != NULL) { - int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; vnodeRelease(pVnode); } @@ -74,7 +75,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { } if (queuedMsgNum == 0) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code}; rpcSendResponse(&rpcRsp); } diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index a5ae8ac83063c599ad4c215bd0a7fa4468810580..775ced09900612da76fff5a60d42839b83fcb133 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -188,6 +188,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { int32_t numOfMsgs; int32_t qtype; + taosBlockSIGPIPE(); dDebug("dnode vwrite worker:%d is running", pWorker->workerId); while (1) { diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index f6307b67d62eb3042c6ef029e461a6bce5a1fc12..c62d5a8207c8cd5f540e3ff861d44cd55c7c8220 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -73,7 +73,8 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { if (*numOfVnodes >= TSDB_MAX_VNODES) { dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES); - continue; + closedir(dir); + return TSDB_CODE_DND_TOO_MANY_VNODES; } else { vnodeList[*numOfVnodes - 1] = vnode; } @@ -245,12 +246,11 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { pStatus->lastReboot = htonl(tsRebootTime); pStatus->numOfCores = htons((uint16_t) tsNumOfCores); pStatus->diskAvailable = tsAvailDataDirGB; - pStatus->alternativeRole = (uint8_t) tsAlternativeRole; + pStatus->alternativeRole = tsAlternativeRole; tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN); // fill cluster cfg parameters pStatus->clusterCfg.numOfMnodes = htonl(tsNumOfMnodes); - pStatus->clusterCfg.enableBalance = htonl(tsEnableBalance); pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum); pStatus->clusterCfg.offlineThreshold = htonl(tsOfflineThreshold); pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); @@ -262,7 +262,12 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); + + pStatus->clusterCfg.enableBalance = tsEnableBalance; + pStatus->clusterCfg.flowCtrl = tsEnableFlowCtrl; + pStatus->clusterCfg.slaveQuery = tsEnableSlaveQuery; + pStatus->clusterCfg.adjustMaster = tsEnableAdjustMaster; vnodeBuildStatusMsg(pStatus); contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); @@ -284,4 +289,4 @@ void dnodeSendStatusMsgToMnode() { dInfo("force send status msg to mnode"); taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer); } -} \ No newline at end of file +} diff --git a/src/inc/dnode.h b/src/inc/dnode.h index dd41360e68e1868dc1b947cfc5ab4188e96fc320..877738778b022c9c7d38a3801beb5cdc86ff9f4d 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -36,6 +36,8 @@ bool dnodeIsMasterEp(char *ep); void dnodeGetEpSetForPeer(SRpcEpSet *epSet); void dnodeGetEpSetForShell(SRpcEpSet *epSet); int32_t dnodeGetDnodeId(); +void dnodeGetClusterId(char *clusterId); + void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); bool dnodeStartMnode(SMInfos *pMinfos); @@ -80,4 +82,4 @@ void dnodeReportStep(char *name, char *desc, int8_t finished); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/inc/mnode.h b/src/inc/mnode.h index bdc30b0c46ced0961715bd48623fdb9e52fb440e..800d767eedac3d4680f74564db00e7645917d13b 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -42,11 +42,12 @@ typedef struct SMnodeMsg { struct SVgObj * pVgroup; struct STableObj *pTable; struct SSTableObj*pSTable; + struct SMnodeMsg *pBatchMasterMsg; SMnodeRsp rpcRsp; - int8_t received; - int8_t successed; - int8_t expected; - int8_t retry; + int16_t received; + int16_t successed; + int16_t expected; + int16_t retry; int32_t incomingTs; int32_t code; void * pObj; @@ -57,6 +58,7 @@ typedef struct SMnodeMsg { void * mnodeCreateMsg(SRpcMsg *pRpcMsg); int32_t mnodeInitMsg(SMnodeMsg *pMsg); void mnodeCleanupMsg(SMnodeMsg *pMsg); +void mnodeDestroySubMsg(SMnodeMsg *pSubMsg); int32_t mnodeInitSystem(); int32_t mnodeStartSystem(); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 0cc06be1dbcafbaba1ab00b041aff7af9db6e445..7f1ed408154e6a99232afce3eb927b64f0412c5c 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -369,6 +369,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_DB_UPDATE 1 #define TSDB_DEFAULT_DB_UPDATE_OPTION 0 +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 1 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 + #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second @@ -432,7 +436,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_HTTP 11 #define TSDB_PORT_ARBITRATOR 12 -#define TSDB_MAX_WAL_SIZE (1024*1024) +#define TSDB_MAX_WAL_SIZE (1024*1024*2) typedef enum { TAOS_QTYPE_RPC = 0, diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 7c7e7ec31ab88a1f048767b9cc4aa486bfa788cd..641b65749937438f14744355136b9b6f0aecef3f 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -67,6 +67,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid re TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Client and server's time is not synchronized") TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION, 0, 0x0016, "Invalid app version") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") @@ -193,6 +194,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length") TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, 0, 0x0404, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, 0, 0x0405, "Too many vnode directories") // vnode TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress") @@ -208,9 +210,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Database memory is full for commit failed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, 0, 0x050C, "Database memory is full for waiting commit") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_DROPPING, 0, 0x050D, "Database is dropping") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_BALANCING, 0, 0x050E, "Database is balancing") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Database write operation denied") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, 0, 0x0513, "Database is syncing") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "Invalid table ID") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 2df243eb3ebe942f9a489c167b164082a941d8a4..2dee6dc3bb20c7397ef435d4b7f37e5861f8891e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -324,6 +324,7 @@ typedef struct { typedef struct { char acctId[TSDB_ACCT_LEN]; char serverVersion[TSDB_VERSION_LEN]; + char clusterId[TSDB_CLUSTER_ID_LEN]; int8_t writeAuth; int8_t superAuth; int8_t reserved1; @@ -475,6 +476,7 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; uint16_t tagCondLen; // tag length in current query + uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx @@ -493,6 +495,7 @@ typedef struct { int32_t tsNumOfBlocks; // ts comp block numbers int32_t tsOrder; // ts comp block order int32_t numOfTags; // number of tags columns involved + int32_t sqlstrLen; // sql query string SColumnInfo colList[]; } SQueryTableMsg; @@ -517,16 +520,17 @@ typedef struct SRetrieveTableRsp { } SRetrieveTableRsp; typedef struct { - int32_t vgId; - int32_t dbCfgVersion; - int64_t totalStorage; - int64_t compStorage; - int64_t pointsWritten; - uint8_t status; - uint8_t role; - uint8_t replica; - uint8_t reserved; - int32_t vgCfgVersion; + int32_t vgId; + int32_t dbCfgVersion; + int64_t totalStorage; + int64_t compStorage; + int64_t pointsWritten; + uint64_t vnodeVersion; + int32_t vgCfgVersion; + uint8_t status; + uint8_t role; + uint8_t replica; + uint8_t reserved; } SVnodeLoad; typedef struct { @@ -549,7 +553,8 @@ typedef struct { int8_t quorum; int8_t ignoreExist; int8_t update; - int8_t reserve[9]; + int8_t cacheLastRow; + int8_t reserve[8]; } SCreateDbMsg, SAlterDbMsg; typedef struct { @@ -604,7 +609,6 @@ typedef struct { typedef struct { int32_t numOfMnodes; // tsNumOfMnodes - int32_t enableBalance; // tsEnableBalance int32_t mnodeEqualVnodeNum; // tsMnodeEqualVnodeNum int32_t offlineThreshold; // tsOfflineThreshold int32_t statusInterval; // tsStatusInterval @@ -615,6 +619,11 @@ typedef struct { int64_t checkTime; // 1970-01-01 00:00:00.000 char locale[TSDB_LOCALE_LEN]; // tsLocale char charset[TSDB_LOCALE_LEN]; // tsCharset + int8_t enableBalance; // tsEnableBalance + int8_t flowCtrl; + int8_t slaveQuery; + int8_t adjustMaster; + int8_t reserved[4]; } SClusterCfg; typedef struct { @@ -657,12 +666,14 @@ typedef struct { int8_t precision; int8_t compression; int8_t walLevel; - int8_t replications; + int8_t vgReplica; int8_t wals; int8_t quorum; int8_t update; - int8_t reserved[11]; + int8_t cacheLastRow; int32_t vgCfgVersion; + int8_t dbReplica; + int8_t reserved[9]; } SVnodeCfg; typedef struct { @@ -716,7 +727,6 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; char tableId[TSDB_TABLE_FNAME_LEN]; // table id - char sTableId[TSDB_TABLE_FNAME_LEN]; uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -726,6 +736,9 @@ typedef struct STableMetaMsg { int32_t tid; uint64_t uid; SVgroupMsg vgroup; + + char sTableName[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; SSchema schema[]; } STableMetaMsg; diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 04d6c7881556defb93b0b65cefff6f69b9bf7cbc..262bf30309ae8bc8bb24b844a88acbfd2290d0c0 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -66,6 +66,7 @@ typedef struct { int8_t precision; int8_t compression; int8_t update; + int8_t cacheLastRow; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS @@ -119,7 +120,7 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg); int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); -TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); +// TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 0a5a3d2fa402c3dd03f1feef8cf05e446922972b..7bd7b228cbbdbfa4d5d6db2d6fe9c0bb647c9fea 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -114,114 +114,115 @@ #define TK_COMP 96 #define TK_PRECISION 97 #define TK_UPDATE 98 -#define TK_LP 99 -#define TK_RP 100 -#define TK_TAGS 101 -#define TK_USING 102 -#define TK_AS 103 -#define TK_COMMA 104 -#define TK_NULL 105 -#define TK_SELECT 106 -#define TK_UNION 107 -#define TK_ALL 108 -#define TK_FROM 109 -#define TK_VARIABLE 110 -#define TK_INTERVAL 111 -#define TK_FILL 112 -#define TK_SLIDING 113 -#define TK_ORDER 114 -#define TK_BY 115 -#define TK_ASC 116 -#define TK_DESC 117 -#define TK_GROUP 118 -#define TK_HAVING 119 -#define TK_LIMIT 120 -#define TK_OFFSET 121 -#define TK_SLIMIT 122 -#define TK_SOFFSET 123 -#define TK_WHERE 124 -#define TK_NOW 125 -#define TK_RESET 126 -#define TK_QUERY 127 -#define TK_ADD 128 -#define TK_COLUMN 129 -#define TK_TAG 130 -#define TK_CHANGE 131 -#define TK_SET 132 -#define TK_KILL 133 -#define TK_CONNECTION 134 -#define TK_STREAM 135 -#define TK_COLON 136 -#define TK_ABORT 137 -#define TK_AFTER 138 -#define TK_ATTACH 139 -#define TK_BEFORE 140 -#define TK_BEGIN 141 -#define TK_CASCADE 142 -#define TK_CLUSTER 143 -#define TK_CONFLICT 144 -#define TK_COPY 145 -#define TK_DEFERRED 146 -#define TK_DELIMITERS 147 -#define TK_DETACH 148 -#define TK_EACH 149 -#define TK_END 150 -#define TK_EXPLAIN 151 -#define TK_FAIL 152 -#define TK_FOR 153 -#define TK_IGNORE 154 -#define TK_IMMEDIATE 155 -#define TK_INITIALLY 156 -#define TK_INSTEAD 157 -#define TK_MATCH 158 -#define TK_KEY 159 -#define TK_OF 160 -#define TK_RAISE 161 -#define TK_REPLACE 162 -#define TK_RESTRICT 163 -#define TK_ROW 164 -#define TK_STATEMENT 165 -#define TK_TRIGGER 166 -#define TK_VIEW 167 -#define TK_COUNT 168 -#define TK_SUM 169 -#define TK_AVG 170 -#define TK_MIN 171 -#define TK_MAX 172 -#define TK_FIRST 173 -#define TK_LAST 174 -#define TK_TOP 175 -#define TK_BOTTOM 176 -#define TK_STDDEV 177 -#define TK_PERCENTILE 178 -#define TK_APERCENTILE 179 -#define TK_LEASTSQUARES 180 -#define TK_HISTOGRAM 181 -#define TK_DIFF 182 -#define TK_SPREAD 183 -#define TK_TWA 184 -#define TK_INTERP 185 -#define TK_LAST_ROW 186 -#define TK_RATE 187 -#define TK_IRATE 188 -#define TK_SUM_RATE 189 -#define TK_SUM_IRATE 190 -#define TK_AVG_RATE 191 -#define TK_AVG_IRATE 192 -#define TK_TBID 193 -#define TK_SEMI 194 -#define TK_NONE 195 -#define TK_PREV 196 -#define TK_LINEAR 197 -#define TK_IMPORT 198 -#define TK_METRIC 199 -#define TK_TBNAME 200 -#define TK_JOIN 201 -#define TK_METRICS 202 -#define TK_STABLE 203 -#define TK_INSERT 204 -#define TK_INTO 205 -#define TK_VALUES 206 +#define TK_CACHELAST 99 +#define TK_LP 100 +#define TK_RP 101 +#define TK_TAGS 102 +#define TK_USING 103 +#define TK_AS 104 +#define TK_COMMA 105 +#define TK_NULL 106 +#define TK_SELECT 107 +#define TK_UNION 108 +#define TK_ALL 109 +#define TK_FROM 110 +#define TK_VARIABLE 111 +#define TK_INTERVAL 112 +#define TK_FILL 113 +#define TK_SLIDING 114 +#define TK_ORDER 115 +#define TK_BY 116 +#define TK_ASC 117 +#define TK_DESC 118 +#define TK_GROUP 119 +#define TK_HAVING 120 +#define TK_LIMIT 121 +#define TK_OFFSET 122 +#define TK_SLIMIT 123 +#define TK_SOFFSET 124 +#define TK_WHERE 125 +#define TK_NOW 126 +#define TK_RESET 127 +#define TK_QUERY 128 +#define TK_ADD 129 +#define TK_COLUMN 130 +#define TK_TAG 131 +#define TK_CHANGE 132 +#define TK_SET 133 +#define TK_KILL 134 +#define TK_CONNECTION 135 +#define TK_STREAM 136 +#define TK_COLON 137 +#define TK_ABORT 138 +#define TK_AFTER 139 +#define TK_ATTACH 140 +#define TK_BEFORE 141 +#define TK_BEGIN 142 +#define TK_CASCADE 143 +#define TK_CLUSTER 144 +#define TK_CONFLICT 145 +#define TK_COPY 146 +#define TK_DEFERRED 147 +#define TK_DELIMITERS 148 +#define TK_DETACH 149 +#define TK_EACH 150 +#define TK_END 151 +#define TK_EXPLAIN 152 +#define TK_FAIL 153 +#define TK_FOR 154 +#define TK_IGNORE 155 +#define TK_IMMEDIATE 156 +#define TK_INITIALLY 157 +#define TK_INSTEAD 158 +#define TK_MATCH 159 +#define TK_KEY 160 +#define TK_OF 161 +#define TK_RAISE 162 +#define TK_REPLACE 163 +#define TK_RESTRICT 164 +#define TK_ROW 165 +#define TK_STATEMENT 166 +#define TK_TRIGGER 167 +#define TK_VIEW 168 +#define TK_COUNT 169 +#define TK_SUM 170 +#define TK_AVG 171 +#define TK_MIN 172 +#define TK_MAX 173 +#define TK_FIRST 174 +#define TK_LAST 175 +#define TK_TOP 176 +#define TK_BOTTOM 177 +#define TK_STDDEV 178 +#define TK_PERCENTILE 179 +#define TK_APERCENTILE 180 +#define TK_LEASTSQUARES 181 +#define TK_HISTOGRAM 182 +#define TK_DIFF 183 +#define TK_SPREAD 184 +#define TK_TWA 185 +#define TK_INTERP 186 +#define TK_LAST_ROW 187 +#define TK_RATE 188 +#define TK_IRATE 189 +#define TK_SUM_RATE 190 +#define TK_SUM_IRATE 191 +#define TK_AVG_RATE 192 +#define TK_AVG_IRATE 193 +#define TK_TBID 194 +#define TK_SEMI 195 +#define TK_NONE 196 +#define TK_PREV 197 +#define TK_LINEAR 198 +#define TK_IMPORT 199 +#define TK_METRIC 200 +#define TK_TBNAME 201 +#define TK_JOIN 202 +#define TK_METRICS 203 +#define TK_STABLE 204 +#define TK_INSERT 205 +#define TK_INTO 206 +#define TK_VALUES 207 #define TK_SPACE 300 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 3dd0c58ae217d38d280d2bd30d85e13b33e876b0..7f6a8d65e7c6153751da2525d09a0a40783e6e91 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -28,7 +28,7 @@ extern "C" { default: \ (_v) = (_finalType)GET_INT32_VAL(_data); \ break; \ - }; + } #ifdef __cplusplus } diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index 66e8cf73988ab25db7544b9a52215d2279630c63..bf5278430054598b31e591ff728412556dabc6f0 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -3,4 +3,5 @@ PROJECT(TDengine) ADD_SUBDIRECTORY(shell) ADD_SUBDIRECTORY(taosdemo) +ADD_SUBDIRECTORY(taosdemox) ADD_SUBDIRECTORY(taosdump) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 627d06ac2e59ec2bdb222998ea05018c038830ab..fca0e93472a350e8a6fddb8acc5ff54af67fa833 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -302,14 +302,12 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); - TAOS_RES* tmpSql = NULL; - TAOS_RES* pSql = taos_query_h(con, command, &tmpSql); + TAOS_RES* pSql = taos_query_h(con, command, &result); if (taos_errno(pSql)) { taos_error(pSql, st); return; } - atomic_store_64(&result, ((SSqlObj*)tmpSql)->self); int64_t oresult = atomic_load_64(&result); if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 53e7d2398450fe22a11da1e7254e0c2ab5f02ea4..1d77a6bb6395d7b171a3d79ee75df27ec1826f34 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -95,7 +95,7 @@ typedef struct DemoArguments { {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, #endif {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, + {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, {0, 's', "sql file", 0, "The select sql file.", 3}, {0, 'M', 0, 0, "Use metric flag.", 13}, @@ -205,10 +205,10 @@ typedef struct DemoArguments { arguments->tb_prefix = arg; break; case 'M': - arguments->use_metric = false; + arguments->use_metric = true; break; case 'x': - arguments->insert_only = false; + arguments->insert_only = true; break; case 'c': if (wordexp(arg, &full_path, 0) != 0) { @@ -406,9 +406,9 @@ typedef struct DemoArguments { } else if (strcmp(argv[i], "-m") == 0) { arguments->tb_prefix = argv[++i]; } else if (strcmp(argv[i], "-M") == 0) { - arguments->use_metric = false; + arguments->use_metric = true; } else if (strcmp(argv[i], "-x") == 0) { - arguments->insert_only = false; + arguments->insert_only = true; } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-O") == 0) { @@ -476,6 +476,14 @@ typedef struct { int notFinished; tsem_t lock_sem; int counter; + + // insert delay statitics + int64_t cntDelay; + int64_t totalDelay; + int64_t avgDelay; + int64_t maxDelay; + int64_t minDelay; + } info; typedef struct { @@ -575,7 +583,7 @@ int main(int argc, char *argv[]) { arguments.num_of_DPT = 100000; arguments.num_of_RPR = 1000; arguments.use_metric = true; - arguments.insert_only = true; + arguments.insert_only = false; // end change parse_args(argc, argv, &arguments); @@ -739,6 +747,9 @@ int main(int argc, char *argv[]) { printf("Inserting data......\n"); pthread_t *pids = malloc(threads * sizeof(pthread_t)); info *infos = malloc(threads * sizeof(info)); + + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(info)); int a = ntables / threads; if (a < 1) { @@ -768,6 +779,7 @@ int main(int argc, char *argv[]) { t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; t_info->counter = 0; + t_info->minDelay = INT16_MAX; tsem_init(&(t_info->mutex_sem), 0, 1); t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; @@ -799,12 +811,29 @@ int main(int argc, char *argv[]) { t, (int64_t)ntables * nrecords_per_table, nrecords_per_request, (int64_t)ntables * nrecords_per_table / t); + int64_t totalDelay = 0; + int64_t maxDelay = 0; + int64_t minDelay = INT16_MAX; + int64_t cntDelay = 0; + double avgDelay = 0; for (int i = 0; i < threads; i++) { info *t_info = infos + i; taos_close(t_info->taos); tsem_destroy(&(t_info->mutex_sem)); tsem_destroy(&(t_info->lock_sem)); + + totalDelay += t_info->totalDelay; + cntDelay += t_info->cntDelay; + if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; + if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; } + avgDelay = (double)totalDelay / cntDelay; + + fprintf(fp, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + + printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); free(pids); free(infos); @@ -859,7 +888,7 @@ int main(int argc, char *argv[]) { } - if (!insert_only) { + if (false == insert_only) { // query data pthread_t read_id; info *rInfo = malloc(sizeof(info)); @@ -998,7 +1027,7 @@ void * createTable(void *sarg) /* Create all the tables; */ printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols); + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s);", winfo->db_name, winfo->tb_prefix, i, winfo->cols); queryDB(winfo->taos, command); } } else { @@ -1204,6 +1233,41 @@ void *readMetric(void *sarg) { return NULL; } +static int queryDbExec(TAOS *taos, char *command, int type) { + int i; + TAOS_RES *res = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } + + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + if (1 == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + void queryDB(TAOS *taos, char *command) { int i; TAOS_RES *pSql = NULL; @@ -1273,7 +1337,21 @@ void *syncWrite(void *sarg) { } /* puts(buffer); */ - queryDB(winfo->taos, buffer); + int64_t startTs; + int64_t endTs; + startTs = taosGetTimestampUs(); + //queryDB(winfo->taos, buffer); + int affectedRows = queryDbExec(winfo->taos, buffer, 1); + + if (0 <= affectedRows){ + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + if (delay > winfo->maxDelay) winfo->maxDelay = delay; + if (delay < winfo->minDelay) winfo->minDelay = delay; + winfo->cntDelay++; + winfo->totalDelay += delay; + //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; + } if (tID == winfo->end_table_id) { i = inserted; diff --git a/src/kit/taosdemox/CMakeLists.txt b/src/kit/taosdemox/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..3f5e725aeaf9fa957e8aa9cdd54978df7fbd94ed --- /dev/null +++ b/src/kit/taosdemox/CMakeLists.txt @@ -0,0 +1,25 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/include) + +IF (TD_LINUX) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(taosdemox ${SRC}) + + #find_program(HAVE_CURL NAMES curl) + IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) + ADD_DEFINITIONS(-DTD_LOWA_CURL) + LINK_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/lib) + ADD_LIBRARY(curl STATIC IMPORTED) + SET_PROPERTY(TARGET curl PROPERTY IMPORTED_LOCATION ${TD_COMMUNITY_DIR}/deps/libcurl/lib/libcurl.a) + TARGET_LINK_LIBRARIES(taosdemox curl) + ENDIF () + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemox taos_static cJson) + ELSE () + TARGET_LINK_LIBRARIES(taosdemox taos cJson) + ENDIF () +ENDIF () diff --git a/src/kit/taosdemox/insert.json b/src/kit/taosdemox/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..88416c13a4cad68a25d090f69f9b3ae7dee66296 --- /dev/null +++ b/src/kit/taosdemox/insert.json @@ -0,0 +1,53 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 2, + "result_file": "./insert_res.txt", + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 2, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 1, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/src/kit/taosdemox/query.json b/src/kit/taosdemox/query.json new file mode 100644 index 0000000000000000000000000000000000000000..53d0b319212196257aa3e84be1221bd6e2bd0d8d --- /dev/null +++ b/src/kit/taosdemox/query.json @@ -0,0 +1,17 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db01", + "super_table_query": + {"rate":1, "concurrent":1, + "sqls": [{"sql": "select count(*) from stb01", "result": "./query_res0.txt"}] + }, + "sub_table_query": + {"stblname": "stb01", "rate":1, "threads":1, + "sqls": [{"sql": "select count(*) from xxxx", "result": "./query_res1.txt"}] + } +} diff --git a/src/kit/taosdemox/subscribe.json b/src/kit/taosdemox/subscribe.json new file mode 100644 index 0000000000000000000000000000000000000000..6dfacdd6ed112b398cf38731147cafc02879efe2 --- /dev/null +++ b/src/kit/taosdemox/subscribe.json @@ -0,0 +1,17 @@ +{ + "filetype":"subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db01", + "super_table_query": + {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", + "sqls": [{"sql": "select avg(c1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] + }, + "sub_table_query": + {"stblname": "stb01", "threads":1, "mode":"sync", "interval":10000, "restart":"yes", "keepProgress":"yes", + "sqls": [{"sql": "select col1 from xxxx where col1 > 10;", "result": "./subscribe_res1.txt"}] + } +} diff --git a/src/kit/taosdemox/taosdemox.c b/src/kit/taosdemox/taosdemox.c new file mode 100644 index 0000000000000000000000000000000000000000..5c9fd025f0a39e03ea8eaedc1bc81bc7e88d8eb4 --- /dev/null +++ b/src/kit/taosdemox/taosdemox.c @@ -0,0 +1,4621 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +/* + when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread. +*/ + +#define _GNU_SOURCE +#define CURL_STATICLIB + +#ifdef TD_LOWA_CURL +#include "curl/curl.h" +#endif + +#ifdef LINUX + #include "os.h" + #include "cJSON.h" + #include + #include + #include + #ifndef _ALPINE + #include + #endif + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include +#else + #include + #include + #include + #include "os.h" + + #pragma comment ( lib, "libcurl.lib" ) + #pragma comment ( lib, "ws2_32.lib" ) + #pragma comment ( lib, "winmm.lib" ) + #pragma comment ( lib, "wldap32.lib" ) +#endif + +#include "taos.h" +#include "tutil.h" + +extern char configDir[]; + +#define INSERT_JSON_NAME "insert.json" +#define QUERY_JSON_NAME "query.json" +#define SUBSCRIBE_JSON_NAME "subscribe.json" + +#define INSERT_MODE 0 +#define QUERY_MODE 1 +#define SUBSCRIBE_MODE 2 + +#define MAX_SQL_SIZE 65536 +#define BUFFER_SIZE (65536*2) +#define MAX_DB_NAME_SIZE 64 +#define MAX_TB_NAME_SIZE 64 +#define MAX_DATA_SIZE 16000 +#define MAX_NUM_DATATYPE 10 +#define OPT_ABORT 1 /* –abort */ +#define STRING_LEN 60000 +#define MAX_PREPARED_RAND 1000000 +//#define MAX_SQL_SIZE 65536 +#define MAX_FILE_NAME_LEN 256 + +#define MAX_SAMPLES_ONCE_FROM_FILE 10000 +#define MAX_NUM_DATATYPE 10 + +#define MAX_DB_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 8 +#define MAX_COLUMN_COUNT 1024 +#define MAX_TAG_COUNT 128 + +#define MAX_QUERY_SQL_COUNT 10 +#define MAX_QUERY_SQL_LENGTH 256 + + +#define MAX_LINE_COUNT_IN_MEM 10000 + +typedef enum CREATE_SUB_TALBE_MOD_EN { + PRE_CREATE_SUBTBL, + AUTO_CREATE_SUBTBL, + NO_CREATE_SUBTBL +} CREATE_SUB_TALBE_MOD_EN; + +typedef enum TALBE_EXISTS_EN { + TBL_ALREADY_EXISTS, + TBL_NO_EXISTS, + TBL_EXISTS_BUTT +} TALBE_EXISTS_EN; + +enum MODE { + SYNC, + ASYNC, + MODE_BUT +}; + +enum QUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} ; + +enum _describe_table_index { + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC +}; + +typedef struct { + char field[TSDB_COL_NAME_LEN + 1]; + char type[16]; + int length; + char note[128]; +} SColDes; + +/* Used by main to communicate with parse_opt. */ +typedef struct SArguments_S { + char * metaFile; + char * host; + uint16_t port; + char * user; + char * password; + char * database; + int replica; + char * tb_prefix; + char * sqlFile; + bool use_metric; + bool insert_only; + char * output_file; + int mode; + char * datatype[MAX_NUM_DATATYPE + 1]; + int len_of_binary; + int num_of_CPR; + int num_of_threads; + int num_of_RPR; + int num_of_tables; + int num_of_DPT; + int abort; + int disorderRatio; + int disorderRange; + int method_of_delete; + char ** arg_list; +} SArguments; + +typedef struct SColumn_S { + char field[TSDB_COL_NAME_LEN + 1]; + char dataType[MAX_TB_NAME_SIZE]; + int dataLen; + char note[128]; +} StrColumn; + +typedef struct SSuperTable_S { + char sTblName[MAX_TB_NAME_SIZE]; + int childTblCount; + bool superTblExists; // 0: no, 1: yes + bool childTblExists; // 0: no, 1: yes + int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table + char childTblPrefix[MAX_TB_NAME_SIZE]; + char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample + char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful + int insertRate; // 0: unlimit > 0 rows/s + + int multiThreadWriteOneTbl; // 0: no, 1: yes + int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl + int rowsPerTbl; // + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms or us by database precision + int maxSqlLen; // + + int64_t insertRows; // 0: no limit + int timeStampStep; + char startTimestamp[MAX_TB_NAME_SIZE]; // + char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFile[MAX_FILE_NAME_LEN]; + char tagsFile[MAX_FILE_NAME_LEN]; + + int columnCount; + StrColumn columns[MAX_COLUMN_COUNT]; + int tagCount; + StrColumn tags[MAX_TAG_COUNT]; + + char* childTblName; + char* colsOfCreatChildTable; + int lenOfOneRow; + int lenOfTagOfOneRow; + + char* sampleDataBuf; + int sampleDataBufSize; + //int sampleRowCount; + //int sampleUsePos; + + int tagSource; // 0: rand, 1: tag sample + char* tagDataBuf; + int tagSampleCount; + int tagUsePos; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; +} SSuperTable; + +typedef struct SDbCfg_S { +// int maxtablesPerVnode; + int minRows; + int maxRows; + int comp; + int walLevel; + int fsync; + int replica; + int update; + int keep; + int days; + int cache; + int blocks; + int quorum; + char precision[MAX_TB_NAME_SIZE]; +} SDbCfg; + +typedef struct SDataBase_S { + char dbName[MAX_DB_NAME_SIZE]; + int drop; // 0: use exists, 1: if exists, drop then new create + SDbCfg dbCfg; + int superTblCount; + SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; +} SDataBase; + +typedef struct SDbs_S { + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_DB_NAME_SIZE]; + uint16_t port; + char user[MAX_DB_NAME_SIZE]; + char password[MAX_DB_NAME_SIZE]; + char resultFile[MAX_FILE_NAME_LEN]; + bool use_metric; + bool insert_only; + bool do_aggreFunc; + bool queryMode; + + int threadCount; + int dbCount; + SDataBase db[MAX_DB_COUNT]; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; +} SDbs; + +typedef struct SuperQueryInfo_S { + int rate; // 0: unlimit > 0 loop/s + int concurrent; + int sqlCount; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int subscribeRestart; + int subscribeKeepProgress; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; +} SuperQueryInfo; + +typedef struct SubQueryInfo_S { + char sTblName[MAX_TB_NAME_SIZE]; + int rate; // 0: unlimit > 0 loop/s + int threadCnt; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int subscribeRestart; + int subscribeKeepProgress; + int childTblCount; + char childTblPrefix[MAX_TB_NAME_SIZE]; + int sqlCount; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + + char* childTblName; +} SubQueryInfo; + +typedef struct SQueryMetaInfo_S { + char cfgDir[MAX_FILE_NAME_LEN]; + char host[MAX_DB_NAME_SIZE]; + uint16_t port; + char user[MAX_DB_NAME_SIZE]; + char password[MAX_DB_NAME_SIZE]; + char dbName[MAX_DB_NAME_SIZE]; + char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful + + SuperQueryInfo superQueryInfo; + SubQueryInfo subQueryInfo; +} SQueryMetaInfo; + +typedef struct SThreadInfo_S { + TAOS *taos; + #ifdef TD_LOWA_CURL + CURL *curl_handle; + #endif + int threadID; + char db_name[MAX_DB_NAME_SIZE]; + char fp[4096]; + char tb_prefix[MAX_TB_NAME_SIZE]; + int start_table_id; + int end_table_id; + int data_of_rate; + int64_t start_time; + char* cols; + bool use_metric; + SSuperTable* superTblInfo; + + // for async insert + tsem_t lock_sem; + int64_t counter; + int64_t st; + int64_t et; + int64_t lastTs; + int nrecords_per_request; + + // statistics + int64_t totalRowsInserted; + int64_t totalAffectedRows; +} threadInfo; + +typedef struct curlMemInfo_S { + char *buf; + size_t sizeleft; + } curlMemInfo; + + + +#ifdef LINUX + /* The options we understand. */ + static struct argp_option options[] = { + {0, 'f', "meta file", 0, "The meta data to the execution procedure, if use -f, all others options invalid. Default is NULL.", 0}, + #ifdef _TD_POWER_ + {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/power/'.", 1}, + {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'powerdb'.", 2}, + #else + {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 1}, + {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 2}, + #endif + {0, 'h', "host", 0, "The host to connect to TDengine. Default is localhost.", 2}, + {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 2}, + {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, + {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, + {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 4}, + {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 4}, + {0, 's', "sql file", 0, "The select sql file.", 6}, + {0, 'M', 0, 0, "Use metric flag.", 4}, + {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 6}, + {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 4}, + {0, 'b', "type_of_cols", 0, "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.", 4}, + {0, 'w', "length_of_chartype", 0, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16", 4}, + {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 10.", 4}, + {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 4}, + // {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 100.", 4}, + {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 4}, + {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 10000.", 4}, + {0, 'x', 0, 0, "Not insert only flag.", 4}, + {0, 'O', "disorderRatio", 0, "Insert mode--0: In order, > 0: disorder ratio. Default is in order.", 4}, + {0, 'R', "disorderRang", 0, "Out of order data's range, ms, default is 1000.", 4}, + //{0, 'D', "delete database", 0, "if elete database if exists. 0: no, 1: yes, default is 1", 5}, + {0}}; + +/* Parse a single option. */ +static error_t parse_opt(int key, char *arg, struct argp_state *state) { + // Get the input argument from argp_parse, which we know is a pointer to our arguments structure. + SArguments *arguments = state->input; + wordexp_t full_path; + char **sptr; + switch (key) { + case 'f': + arguments->metaFile = arg; + break; + case 'h': + arguments->host = arg; + break; + case 'p': + arguments->port = atoi(arg); + break; + case 'u': + arguments->user = arg; + break; + case 'P': + arguments->password = arg; + break; + case 'o': + arguments->output_file = arg; + break; + case 's': + arguments->sqlFile = arg; + break; + case 'q': + arguments->mode = atoi(arg); + break; + case 'T': + arguments->num_of_threads = atoi(arg); + break; + //case 'r': + // arguments->num_of_RPR = atoi(arg); + // break; + case 't': + arguments->num_of_tables = atoi(arg); + break; + case 'n': + arguments->num_of_DPT = atoi(arg); + break; + case 'd': + arguments->database = arg; + break; + case 'l': + arguments->num_of_CPR = atoi(arg); + break; + case 'b': + sptr = arguments->datatype; + if (strstr(arg, ",") == NULL) { + if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && + strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && + strcasecmp(arg, "SMALLINT") != 0 && strcasecmp(arg, "TIMESTAMP") != 0 && + strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && + strcasecmp(arg, "BINARY") != 0 && strcasecmp(arg, "NCHAR") != 0) { + argp_error(state, "Invalid data_type!"); + } + sptr[0] = arg; + } else { + int index = 0; + char *dupstr = strdup(arg); + char *running = dupstr; + char *token = strsep(&running, ","); + while (token != NULL) { + if (strcasecmp(token, "INT") != 0 && strcasecmp(token, "FLOAT") != 0 && + strcasecmp(token, "TINYINT") != 0 && strcasecmp(token, "BOOL") != 0 && + strcasecmp(token, "SMALLINT") != 0 && strcasecmp(token, "TIMESTAMP") != 0 && + strcasecmp(token, "BIGINT") != 0 && strcasecmp(token, "DOUBLE") != 0 && + strcasecmp(token, "BINARY") != 0 && strcasecmp(token, "NCHAR") != 0) { + argp_error(state, "Invalid data_type!"); + } + sptr[index++] = token; + token = strsep(&running, ","); + if (index >= MAX_NUM_DATATYPE) break; + } + } + break; + case 'w': + arguments->len_of_binary = atoi(arg); + break; + case 'm': + arguments->tb_prefix = arg; + break; + case 'M': + arguments->use_metric = true; + break; + case 'x': + arguments->insert_only = false; + break; + case 'c': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", arg); + return -1; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + break; + case 'O': + arguments->disorderRatio = atoi(arg); + if (arguments->disorderRatio < 0 || arguments->disorderRatio > 100) + { + argp_error(state, "Invalid disorder ratio, should 1 ~ 100!"); + } + break; + case 'R': + arguments->disorderRange = atoi(arg); + break; + case 'a': + arguments->replica = atoi(arg); + if (arguments->replica > 3 || arguments->replica < 1) + { + arguments->replica = 1; + } + break; + //case 'D': + // arguments->method_of_delete = atoi(arg); + // break; + case OPT_ABORT: + arguments->abort = 1; + break; + case ARGP_KEY_ARG: + /*arguments->arg_list = &state->argv[state->next-1]; + state->next = state->argc;*/ + argp_usage(state); + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static struct argp argp = {options, parse_opt, 0, 0}; + +void parse_args(int argc, char *argv[], SArguments *arguments) { + argp_parse(&argp, argc, argv, 0, 0, arguments); + if (arguments->abort) { + #ifndef _ALPINE + error(10, 0, "ABORTED"); + #else + abort(); + #endif + } +} + +#else + void printHelp() { + char indent[10] = " "; + printf("%s%s\n", indent, "-f"); + printf("%s%s%s\n", indent, indent, "The meta file to the execution procedure. Default is './meta.json'."); + printf("%s%s\n", indent, "-c"); + printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); + } + + void parse_args(int argc, char *argv[], SArguments *arguments) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-f") == 0) { + arguments->metaFile = argv[++i]; + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "wrong options\n"); + printHelp(); + exit(EXIT_FAILURE); + } + } + } +#endif + +static bool getInfoFromJsonFile(char* file); +//static int generateOneRowDataForStb(SSuperTable* stbInfo); +//static int getDataIntoMemForStb(SSuperTable* stbInfo); +static void init_rand_data(); +static int createDatabases(); +static void createChildTables(); +static int queryDbExec(TAOS *taos, char *command, int type); + +/* ************ Global variables ************ */ + +int32_t randint[MAX_PREPARED_RAND]; +int64_t randbigint[MAX_PREPARED_RAND]; +float randfloat[MAX_PREPARED_RAND]; +double randdouble[MAX_PREPARED_RAND]; +char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; + +SArguments g_args = {NULL, + "127.0.0.1", // host + 6030, // port + "root", // user + #ifdef _TD_POWER_ + "powerdb", // password + #else + "taosdata", // password + #endif + "test", // database + 1, // replica + "t", // tb_prefix + NULL, // sqlFile + false, // use_metric + true, // insert_only + "./output.txt", // output_file + 0, // mode : sync or async + { + "TINYINT", // datatype + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "BINARY", + "NCHAR", + "BOOL", + "TIMESTAMP" + }, + 16, // len_of_binary + 10, // num_of_CPR + 10, // num_of_connections/thread + 100, // num_of_RPR + 10000, // num_of_tables + 10000, // num_of_DPT + 0, // abort + 0, // disorderRatio + 1000, // disorderRange + 1, // method_of_delete + NULL // arg_list +}; + + +static int g_jsonType = 0; +static SDbs g_Dbs; +static int g_totalChildTables = 0; +static SQueryMetaInfo g_queryInfo; +static FILE * g_fpOfInsertResult = NULL; + + +void tmfclose(FILE *fp) { + if (NULL != fp) { + fclose(fp); + } +} + +void tmfree(char *buf) { + if (NULL != buf) { + free(buf); + } +} + +static int queryDbExec(TAOS *taos, char *command, int type) { + int i; + TAOS_RES *res = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } + + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + +static void getResult(TAOS_RES *res, char* resultFileName) { + TAOS_ROW row = NULL; + int num_rows = 0; + int num_fields = taos_field_count(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + + FILE *fp = NULL; + if (resultFileName[0] != 0) { + fp = fopen(resultFileName, "at"); + if (fp == NULL) { + fprintf(stderr, "failed to open result file: %s, result will not save to file\n", resultFileName); + } + } + + char* databuf = (char*) calloc(1, 100*1024*1024); + if (databuf == NULL) { + fprintf(stderr, "failed to malloc, warning: save result to file slowly!\n"); + return ; + } + + int totalLen = 0; + char temp[16000]; + + // fetch the records row by row + while ((row = taos_fetch_row(res))) { + if (totalLen >= 100*1024*1024 - 32000) { + if (fp) fprintf(fp, "%s", databuf); + totalLen = 0; + memset(databuf, 0, 100*1024*1024); + } + num_rows++; + int len = taos_print_row(temp, row, fields, num_fields); + len += sprintf(temp + len, "\n"); + //printf("query result:%s\n", temp); + memcpy(databuf + totalLen, temp, len); + totalLen += len; + } + + if (fp) fprintf(fp, "%s", databuf); + tmfclose(fp); + free(databuf); +} + +static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { + TAOS_RES *res = taos_query(taos, command); + if (res == NULL || taos_errno(res) != 0) { + printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res)); + taos_free_result(res); + return; + } + + getResult(res, resultFileName); + taos_free_result(res); +} + +double getCurrentTime() { + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { + perror("Failed to get current time in ms"); + return 0.0; + } + + return tv.tv_sec + tv.tv_usec / 1E6; +} + +static int32_t rand_bool(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 2; +} + +static int32_t rand_tinyint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 128; +} + +static int32_t rand_smallint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 32767; +} + +static int32_t rand_int(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor]; +} + +static int64_t rand_bigint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randbigint[cursor]; + +} + +static float rand_float(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randfloat[cursor]; +} + +static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; +void rand_string(char *str, int size) { + str[0] = 0; + if (size > 0) { + //--size; + int n; + for (n = 0; n < size; n++) { + int key = rand_tinyint() % (int)(sizeof(charset) - 1); + str[n] = charset[key]; + } + str[n] = 0; + } +} + +static double rand_double() { + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randdouble[cursor]; + +} + +static void init_rand_data() { + for (int i = 0; i < MAX_PREPARED_RAND; i++){ + randint[i] = (int)(rand() % 65535); + randbigint[i] = (int64_t)(rand() % 2147483648); + randfloat[i] = (float)(rand() / 1000.0); + randdouble[i] = (double)(rand() / 1000000.0); + } +} + +static void printfInsertMeta() { + printf("\033[1m\033[40;32m================ insert.json parse result START ================\033[0m\n"); + printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("user: \033[33m%s\033[0m\n", g_Dbs.user); + printf("password: \033[33m%s\033[0m\n", g_Dbs.password); + printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); + printf("thread count: \033[33m%d\033[0m\n", g_Dbs.threadCount); + + printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + printf("database[\033[33m%d\033[0m]:\n", i); + printf(" database name: \033[33m%s\033[0m\n", g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + printf(" drop: \033[33mno\033[0m\n"); + }else { + printf(" drop: \033[33myes\033[0m\n"); + } + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); + } else { + printf(" precision error: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); + exit(EXIT_FAILURE); + } + } + + printf(" super table count: \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + printf(" super table[\033[33m%d\033[0m]:\n", j); + + printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" childTblExists: \033[33m%s\033[0m\n", "error"); + } + + printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); + printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); + printf(" insertRate: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].insertRate); + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); + }else { + printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); + } + printf(" numberOfTblInOneSql: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); + printf(" rowsPerTbl: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); + printf(" disorderRange: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRange); + printf(" disorderRatio: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRatio); + printf(" maxSqlLen: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + printf(" timeStampStep: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].timeStampStep); + printf(" startTimestamp: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].startTimestamp); + printf(" sampleFormat: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFormat); + printf(" sampleFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFile); + printf(" tagsFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].tagsFile); + + printf(" columnCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + printf("column[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + printf("\n"); + + printf(" tagCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + printf("tag[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + printf("\n"); + } + printf("\n"); + } + printf("\033[1m\033[40;32m================ insert.json parse result END================\033[0m\n"); +} + +static void printfInsertMetaToFile(FILE* fp) { + fprintf(fp, "================ insert.json parse result START================\n"); + fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); + fprintf(fp, "user: %s\n", g_Dbs.user); + fprintf(fp, "password: %s\n", g_Dbs.password); + fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); + fprintf(fp, "thread count: %d\n", g_Dbs.threadCount); + + fprintf(fp, "database count: %d\n", g_Dbs.dbCount); + for (int i = 0; i < g_Dbs.dbCount; i++) { + fprintf(fp, "database[%d]:\n", i); + fprintf(fp, " database name: %s\n", g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + fprintf(fp, " drop: no\n"); + }else { + fprintf(fp, " drop: yes\n"); + } + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + fprintf(fp, " blocks: %d\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + fprintf(fp, " cache: %d\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + fprintf(fp, " days: %d\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + fprintf(fp, " keep: %d\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + fprintf(fp, " replica: %d\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + fprintf(fp, " update: %d\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + fprintf(fp, " minRows: %d\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + fprintf(fp, " maxRows: %d\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + fprintf(fp, " comp: %d\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + fprintf(fp, " walLevel: %d\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + fprintf(fp, " fsync: %d\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + fprintf(fp, " quorum: %d\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + } else { + fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + } + } + + fprintf(fp, " super table count: %d\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + fprintf(fp, " super table[%d]:\n", j); + + fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "yes"); + } else { + fprintf(fp, " autoCreateTable: %s\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "yes"); + } else { + fprintf(fp, " childTblExists: %s\n", "error"); + } + + fprintf(fp, " childTblCount: %d\n", g_Dbs.db[i].superTbls[j].childTblCount); + fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); + fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); + fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); + fprintf(fp, " insertRate: %d\n", g_Dbs.db[i].superTbls[j].insertRate); + fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + fprintf(fp, " multiThreadWriteOneTbl: no\n"); + }else { + fprintf(fp, " multiThreadWriteOneTbl: yes\n"); + } + fprintf(fp, " numberOfTblInOneSql: %d\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); + fprintf(fp, " rowsPerTbl: %d\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); + fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); + fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); + fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + + fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { + fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + fprintf(fp, "\n"); + + fprintf(fp, " tagCount: %d\n ", g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + fprintf(fp, "\n"); + } + fprintf(fp, "\n"); + } + fprintf(fp, "================ insert.json parse result END ================\n\n"); +} + +static void printfQueryMeta() { + printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); + printf("host: \033[33m%s:%u\033[0m\n", g_queryInfo.host, g_queryInfo.port); + printf("user: \033[33m%s\033[0m\n", g_queryInfo.user); + printf("password: \033[33m%s\033[0m\n", g_queryInfo.password); + printf("database name: \033[33m%s\033[0m\n", g_queryInfo.dbName); + + printf("\n"); + printf("super table query info: \n"); + printf("rate: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent); + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); + + if (SUBSCRIBE_MODE == g_jsonType) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); + } + printf("\n"); + printf("sub table query info: \n"); + printf("rate: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.rate); + printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.threadCnt); + printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount); + printf("childTblPrefix: \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.childTblPrefix); + + if (SUBSCRIBE_MODE == g_jsonType) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeKeepProgress); + } + + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.sqlCount); + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]); + } + printf("\n"); + printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); +} + +#ifdef TD_LOWA_CURL +static size_t responseCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + curlMemInfo* mem = (curlMemInfo*)userp; + + char *ptr = realloc(mem->buf, mem->sizeleft + realsize + 1); + if(ptr == NULL) { + /* out of memory! */ + printf("not enough memory (realloc returned NULL)\n"); + return 0; + } + + mem->buf = ptr; + memcpy(&(mem->buf[mem->sizeleft]), contents, realsize); + mem->sizeleft += realsize; + mem->buf[mem->sizeleft] = 0; + + //printf("result:%s\n\n", mem->buf); + + return realsize; +} + +void curlProceLogin(void) +{ + CURL *curl_handle; + CURLcode res; + + curlMemInfo chunk; + + chunk.buf = malloc(1); /* will be grown as needed by the realloc above */ + chunk.sizeleft = 0; /* no data at this point */ + + //curl_global_init(CURL_GLOBAL_ALL); + + /* init the curl session */ + curl_handle = curl_easy_init(); + + curl_easy_setopt(curl_handle,CURLOPT_POSTFIELDS,""); + curl_easy_setopt(curl_handle, CURLOPT_POST, 1); + + char dstUrl[128] = {0}; + snprintf(dstUrl, 128, "http://%s:6041/rest/login/root/taosdata", g_Dbs.host); + + /* specify URL to get */ + curl_easy_setopt(curl_handle, CURLOPT_URL, dstUrl); + + /* send all data to this function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, responseCallback); + + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk); + + /* do it! */ + res = curl_easy_perform(curl_handle); + + /* check for errors */ + if(res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + } + else { + //printf("response len:%lu, content: %s \n", (unsigned long)chunk.sizeleft, chunk.buf); + ; + } + + /* cleanup curl stuff */ + curl_easy_cleanup(curl_handle); + + free(chunk.buf); + + /* we're done with libcurl, so clean it up */ + //curl_global_cleanup(); + + return; +} + +int curlProceSql(char* host, uint16_t port, char* sqlstr, CURL *curl_handle) +{ + //curlProceLogin(); + + //CURL *curl_handle; + CURLcode res; + + curlMemInfo chunk; + + chunk.buf = malloc(1); /* will be grown as needed by the realloc above */ + chunk.sizeleft = 0; /* no data at this point */ + + + char dstUrl[128] = {0}; + snprintf(dstUrl, 128, "http://%s:%u/rest/sql", host, port+TSDB_PORT_HTTP); + + //curl_global_init(CURL_GLOBAL_ALL); + + /* init the curl session */ + //curl_handle = curl_easy_init(); + + //curl_easy_setopt(curl_handle,CURLOPT_POSTFIELDS,""); + curl_easy_setopt(curl_handle, CURLOPT_POST, 1L); + + /* specify URL to get */ + curl_easy_setopt(curl_handle, CURLOPT_URL, dstUrl); + + /* enable TCP keep-alive for this transfer */ + curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPALIVE, 1L); + /* keep-alive idle time to 120 seconds */ + curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPIDLE, 120L); + /* interval time between keep-alive probes: 60 seconds */ + curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPINTVL, 60L); + + /* send all data to this function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, responseCallback); + + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk); + + struct curl_slist *list = NULL; + list = curl_slist_append(list, "Authorization: Basic cm9vdDp0YW9zZGF0YQ=="); + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); + + /* Set the expected upload size. */ + curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)strlen(sqlstr)); + curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, sqlstr); + + /* get it! */ + res = curl_easy_perform(curl_handle); + + /* check for errors */ + if(res != CURLE_OK) { + fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); + return -1; + } + else { + /* curl_easy_perform() block end and return result */ + //printf("[%32.32s] sql response len:%lu, content: %s \n\n", sqlstr, (unsigned long)chunk.sizeleft, chunk.buf); + ; + } + + curl_slist_free_all(list); /* free the list again */ + + /* cleanup curl stuff */ + //curl_easy_cleanup(curl_handle); + + free(chunk.buf); + + /* we're done with libcurl, so clean it up */ + //curl_global_cleanup(); + + return 0; +} +#endif + +char* getTagValueFromTagSample( SSuperTable* stbInfo, int tagUsePos) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos); + + return dataBuf; +} + +char* generateTagVaulesForStb(SSuperTable* stbInfo) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "("); + for (int i = 0; i < stbInfo->tagCount; i++) { + if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", 5))) { + if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) { + printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); + tmfree(dataBuf); + return NULL; + } + + char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); + tmfree(dataBuf); + return NULL; + } + rand_string(buf, stbInfo->tags[i].dataLen); + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", 3)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", 6)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", 5)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", 6)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", 8)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", 7)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", 4)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", 4)) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); + } else { + printf("No support data type: %s\n", stbInfo->tags[i].dataType); + tmfree(dataBuf); + return NULL; + } + } + dataLen -= 2; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); + return dataBuf; +} + +static int calcRowLen(SSuperTable* superTbls) { + int colIndex; + int lenOfOneRow = 0; + + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + lenOfOneRow += 21; + } else { + printf("get error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + + int tagIndex; + int lenOfTagOfOneRow = 0; + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + printf("get error tag type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + return 0; +} + + +static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, int* childTblCountOfSuperTbl) { + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + char* childTblName = *childTblNameOfSuperTbl; + + //get all child table name use cmd: select tbname from superTblName; + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s", dbName, sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + + int childTblCount = 10000; + count = 0; + childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); + char* pTblName = childTblName; + while ((row = taos_fetch_row(res)) != NULL) { + strncpy(pTblName, (char *)row[0], TSDB_TABLE_NAME_LEN); + //printf("==== sub table name: %s\n", pTblName); + count++; + if (count == childTblCount) { + char *tmp = realloc(childTblName, (size_t)count*1.5*TSDB_TABLE_NAME_LEN); + if (tmp != NULL) { + childTblName = tmp; + memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, (size_t)(count*0.5*TSDB_TABLE_NAME_LEN)); + } else { + // exit, if allocate more memory failed + printf("realloc fail for save child table name of %s.%s\n", dbName, sTblName); + tmfree(childTblName); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + } + pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; + } + + *childTblCountOfSuperTbl = count; + *childTblNameOfSuperTbl = childTblName; + + taos_free_result(res); + return 0; +} + +static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + //get schema use cmd: describe superTblName; + snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + return -1; + } + + int tagIndex = 0; + int columnIndex = 0; + TAOS_FIELD *fields = taos_fetch_fields(res); + while ((row = taos_fetch_row(res)) != NULL) { + if (0 == count) { + count++; + continue; + } + + if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { + strncpy(superTbls->tags[tagIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + strncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + strncpy(superTbls->tags[tagIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + tagIndex++; + } else { + strncpy(superTbls->columns[columnIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + strncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + strncpy(superTbls->columns[columnIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + columnIndex++; + } + count++; + } + + superTbls->columnCount = columnIndex; + superTbls->tagCount = tagIndex; + taos_free_result(res); + + calcRowLen(superTbls); + + if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { + //get all child table name use cmd: select tbname from superTblName; + getAllChildNameOfSuperTable(taos, dbName, superTbls->sTblName, &superTbls->childTblName, &superTbls->childTblCount); + } + return 0; +} + +static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { + char command[BUFFER_SIZE] = "\0"; + + char cols[STRING_LEN] = "\0"; + int colIndex; + int len = 0; + + int lenOfOneRow = 0; + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "BINARY", superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "NCHAR", superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT"); + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT"); + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE"); + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP"); + lenOfOneRow += 21; + } else { + taos_close(taos); + printf("config error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); + + // save for creating child table + superTbls->colsOfCreatChildTable = (char*)calloc(len+20, 1); + if (NULL == superTbls->colsOfCreatChildTable) { + printf("Failed when calloc, size:%d", len+1); + taos_close(taos); + exit(-1); + } + snprintf(superTbls->colsOfCreatChildTable, len+20, "(ts timestamp%s)", cols); + + if (use_metric) { + char tags[STRING_LEN] = "\0"; + int tagIndex; + len = 0; + + int lenOfTagOfOneRow = 0; + len += snprintf(tags + len, STRING_LEN - len, "("); + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "BINARY", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "NCHAR", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "INT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BIGINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "SMALLINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "TINYINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BOOL"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "FLOAT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "DOUBLE"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + taos_close(taos); + printf("config error tag type : %s\n", dataType); + exit(-1); + } + } + len -= 2; + len += snprintf(tags + len, STRING_LEN - len, ")"); + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s (ts timestamp%s) tags %s", dbName, superTbls->sTblName, cols, tags); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + return -1; + } + printf("\ncreate supertable %s success!\n\n", superTbls->sTblName); + } + return 0; +} + + +static int createDatabases() { + TAOS * taos = NULL; + int ret = 0; + taos_init(); + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + char command[BUFFER_SIZE] = "\0"; + + + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].drop) { + sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + taos_close(taos); + return -1; + } + } + + int dataLen = 0; + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "create database if not exists %s ", g_Dbs.db[i].dbName); + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "blocks %d ", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "cache %d ", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "days %d ", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "keep %d ", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "replica %d ", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "update %d ", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "minrows %d ", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "maxrows %d ", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "comp %d ", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "wal %d ", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "fsync %d ", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } + + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + taos_close(taos); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); + + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // describe super table, if exists + sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { + g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS; + ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); + } else { + g_Dbs.db[i].superTbls[j].superTblExists = TBL_ALREADY_EXISTS; + ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j]); + } + + if (0 != ret) { + taos_close(taos); + return -1; + } + } + } + + taos_close(taos); + return 0; +} + + +void * createTable(void *sarg) +{ + char command[BUFFER_SIZE] = "\0"; + + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + int64_t lastPrintTime = taosGetTimestampMs(); + + //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); + for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { + if (0 == g_Dbs.use_metric) { + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable); + } else { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + return NULL; + } + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.%s tags %s;", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + free(tagsValBuf); + } + + if (0 != queryDbExec(winfo->taos, command, NO_INSERT_TYPE)){ + return NULL; + } + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] already create %d - %d tables\n", winfo->threadID, winfo->start_table_id, i); + lastPrintTime = currentPrintTime; + } + } + + return NULL; +} + +void startMultiThreadCreateChildTable(char* cols, int threads, int ntables, char* db_name, SSuperTable* superTblInfo) { + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed\n"); + exit(-1); + } + + if (threads < 1) { + threads = 1; + } + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + b = ntables % threads; + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->superTblInfo = superTblInfo; + t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + last = t_info->end_table_id + 1; + t_info->use_metric = 1; + t_info->cols = cols; + pthread_create(pids + i, NULL, createTable, t_info); + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + taos_close(t_info->taos); + } + + free(pids); + free(infos); +} + + +static void createChildTables() { + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + startMultiThreadCreateChildTable(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, g_Dbs.threadCount, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + } + } +} + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +int readTagFromCsvFileToMem(SSuperTable * supterTblInfo) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + + FILE *fp = fopen(supterTblInfo->tagsFile, "r"); + if (fp == NULL) { + printf("Failed to open tags file: %s, reason:%s\n", supterTblInfo->tagsFile, strerror(errno)); + return -1; + } + + if (supterTblInfo->tagDataBuf) { + free(supterTblInfo->tagDataBuf); + supterTblInfo->tagDataBuf = NULL; + } + + supterTblInfo->tagDataBuf = calloc(supterTblInfo->lenOfTagOfOneRow * MAX_LINE_COUNT_IN_MEM, 1); + if (supterTblInfo->tagDataBuf == NULL) { + printf("Failed to calloc, reason:%s\n", strerror(errno)); + fclose(fp); + return -1; + } + + while ((readLen = getline(&line, &n, fp)) != -1) { + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + memcpy(supterTblInfo->tagDataBuf + supterTblInfo->tagSampleCount * supterTblInfo->lenOfTagOfOneRow, line, readLen); + supterTblInfo->tagSampleCount++; + + if (supterTblInfo->tagSampleCount >= MAX_LINE_COUNT_IN_MEM) { + break; + } + } + + free(line); + fclose(fp); + return 0; +} + +int readSampleFromJsonFileToMem(SSuperTable * supterTblInfo) { + // TODO + return 0; +} + + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sampleBuf) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + int getRows = 0; + + memset(sampleBuf, 0, MAX_SAMPLES_ONCE_FROM_FILE* superTblInfo->lenOfOneRow); + while (1) { + readLen = getline(&line, &n, fp); + if (-1 == readLen) { + if(0 != fseek(fp, 0, SEEK_SET)) { + printf("Failed to fseek file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); + return -1; + } + continue; + } + + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + if (readLen > superTblInfo->lenOfOneRow) { + printf("sample row len[%d] overflow define schema len[%d], so discard this row\n", (int32_t)readLen, superTblInfo->lenOfOneRow); + continue; + } + + memcpy(sampleBuf + getRows * superTblInfo->lenOfOneRow, line, readLen); + getRows++; + + if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { + break; + } + } + + tmfree(line); + return 0; +} + +/* +void readSampleFromFileToMem(SSuperTable * supterTblInfo) { + int ret; + if (0 == strncasecmp(supterTblInfo->sampleFormat, "csv", 3)) { + ret = readSampleFromCsvFileToMem(supterTblInfo); + } else if (0 == strncasecmp(supterTblInfo->sampleFormat, "json", 4)) { + ret = readSampleFromJsonFileToMem(supterTblInfo); + } + + if (0 != ret) { + exit(-1); + } +} +*/ +static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* superTbls) { + bool ret = false; + + // columns + cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); + if (columns && columns->type != cJSON_Array) { + printf("failed to read json, columns not found\n"); + goto PARSE_OVER; + } else if (NULL == columns) { + superTbls->columnCount = 0; + superTbls->tagCount = 0; + return true; + } + + int columnSize = cJSON_GetArraySize(columns); + if (columnSize > MAX_COLUMN_COUNT) { + printf("failed to read json, column size overflow, max column size is %d\n", MAX_COLUMN_COUNT); + goto PARSE_OVER; + } + + int count = 1; + int index = 0; + StrColumn columnCase; + + //superTbls->columnCount = columnSize; + for (int k = 0; k < columnSize; ++k) { + cJSON* column = cJSON_GetArrayItem(columns, k); + if (column == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(column, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("failed to read json, column count not found"); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(column, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + printf("failed to read json, column type not found"); + goto PARSE_OVER; + } + //strncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + strncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(column, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + printf("failed to read json, column len not found"); + goto PARSE_OVER; + } else { + columnCase.dataLen = 8; + } + + for (int n = 0; n < count; ++n) { + strncpy(superTbls->columns[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->columns[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->columnCount = index; + + count = 1; + index = 0; + // tags + cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); + if (!tags || tags->type != cJSON_Array) { + printf("failed to read json, tags not found"); + goto PARSE_OVER; + } + + int tagSize = cJSON_GetArraySize(tags); + if (tagSize > MAX_TAG_COUNT) { + printf("failed to read json, tags size overflow, max tag size is %d\n", MAX_TAG_COUNT); + goto PARSE_OVER; + } + + //superTbls->tagCount = tagSize; + for (int k = 0; k < tagSize; ++k) { + cJSON* tag = cJSON_GetArrayItem(tags, k); + if (tag == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(tag, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("failed to read json, column count not found"); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(tag, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + printf("failed to read json, tag type not found"); + goto PARSE_OVER; + } + strncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + printf("failed to read json, column len not found"); + goto PARSE_OVER; + } else { + columnCase.dataLen = 0; + } + + for (int n = 0; n < count; ++n) { + strncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->tags[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->tagCount = index; + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromInsertJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + strncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + strncpy(g_Dbs.host, host->valuestring, MAX_DB_NAME_SIZE); + } else if (!host) { + strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_Dbs.port = port->valueint; + } else if (!port) { + g_Dbs.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + strncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE); + } else if (!user) { + strncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE); + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + strncpy(g_Dbs.password, password->valuestring, MAX_DB_NAME_SIZE); + } else if (!password) { + strncpy(g_Dbs.password, "taosdata", MAX_DB_NAME_SIZE); + } + + cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); + if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { + strncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); + } else if (!resultfile) { + strncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); + } + + cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); + if (threads && threads->type == cJSON_Number) { + g_Dbs.threadCount = threads->valueint; + } else if (!threads) { + g_Dbs.threadCount = 1; + } else { + printf("failed to read json, threads not found"); + goto PARSE_OVER; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (!dbs || dbs->type != cJSON_Array) { + printf("failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + int dbSize = cJSON_GetArraySize(dbs); + if (dbSize > MAX_DB_COUNT) { + printf("failed to read json, databases size overflow, max database is %d\n", MAX_DB_COUNT); + goto PARSE_OVER; + } + + g_Dbs.dbCount = dbSize; + for (int i = 0; i < dbSize; ++i) { + cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); + if (dbinfos == NULL) continue; + + // dbinfo + cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); + if (!dbinfo || dbinfo->type != cJSON_Object) { + printf("failed to read json, dbinfo not found"); + goto PARSE_OVER; + } + + cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); + if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { + printf("failed to read json, db name not found"); + goto PARSE_OVER; + } + strncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); + + cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); + if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { + if (0 == strncasecmp(drop->valuestring, "yes", 3)) { + g_Dbs.db[i].drop = 1; + } else { + g_Dbs.db[i].drop = 0; + } + } else if (!drop) { + g_Dbs.db[i].drop = 0; + } else { + printf("failed to read json, drop not found"); + goto PARSE_OVER; + } + + cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); + if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { + strncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, MAX_DB_NAME_SIZE); + } else if (!precision) { + //strncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, precision not found"); + goto PARSE_OVER; + } + + cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); + if (update && update->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.update = update->valueint; + } else if (!update) { + g_Dbs.db[i].dbCfg.update = -1; + } else { + printf("failed to read json, update not found"); + goto PARSE_OVER; + } + + cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); + if (replica && replica->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.replica = replica->valueint; + } else if (!replica) { + g_Dbs.db[i].dbCfg.replica = -1; + } else { + printf("failed to read json, replica not found"); + goto PARSE_OVER; + } + + cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); + if (keep && keep->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.keep = keep->valueint; + } else if (!keep) { + g_Dbs.db[i].dbCfg.keep = -1; + } else { + printf("failed to read json, keep not found"); + goto PARSE_OVER; + } + + cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); + if (days && days->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.days = days->valueint; + } else if (!days) { + g_Dbs.db[i].dbCfg.days = -1; + } else { + printf("failed to read json, days not found"); + goto PARSE_OVER; + } + + cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); + if (cache && cache->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cache = cache->valueint; + } else if (!cache) { + g_Dbs.db[i].dbCfg.cache = -1; + } else { + printf("failed to read json, cache not found"); + goto PARSE_OVER; + } + + cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); + if (blocks && blocks->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.blocks = blocks->valueint; + } else if (!blocks) { + g_Dbs.db[i].dbCfg.blocks = -1; + } else { + printf("failed to read json, block not found"); + goto PARSE_OVER; + } + + //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); + //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; + //} else if (!maxtablesPerVnode) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; + //} else { + // printf("failed to read json, maxtablesPerVnode not found"); + // goto PARSE_OVER; + //} + + cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); + if (minRows && minRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.minRows = minRows->valueint; + } else if (!minRows) { + g_Dbs.db[i].dbCfg.minRows = -1; + } else { + printf("failed to read json, minRows not found"); + goto PARSE_OVER; + } + + cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); + if (maxRows && maxRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; + } else if (!maxRows) { + g_Dbs.db[i].dbCfg.maxRows = -1; + } else { + printf("failed to read json, maxRows not found"); + goto PARSE_OVER; + } + + cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); + if (comp && comp->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.comp = comp->valueint; + } else if (!comp) { + g_Dbs.db[i].dbCfg.comp = -1; + } else { + printf("failed to read json, comp not found"); + goto PARSE_OVER; + } + + cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); + if (walLevel && walLevel->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; + } else if (!walLevel) { + g_Dbs.db[i].dbCfg.walLevel = -1; + } else { + printf("failed to read json, walLevel not found"); + goto PARSE_OVER; + } + + cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); + if (quorum && quorum->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.quorum = quorum->valueint; + } else if (!quorum) { + g_Dbs.db[i].dbCfg.quorum = -1; + } else { + printf("failed to read json, walLevel not found"); + goto PARSE_OVER; + } + + cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); + if (fsync && fsync->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.fsync = fsync->valueint; + } else if (!fsync) { + g_Dbs.db[i].dbCfg.fsync = -1; + } else { + printf("failed to read json, fsync not found"); + goto PARSE_OVER; + } + + // super_talbes + cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); + if (!stables || stables->type != cJSON_Array) { + printf("failed to read json, super_tables not found"); + goto PARSE_OVER; + } + + int stbSize = cJSON_GetArraySize(stables); + if (stbSize > MAX_SUPER_TABLE_COUNT) { + printf("failed to read json, databases size overflow, max database is %d\n", MAX_SUPER_TABLE_COUNT); + goto PARSE_OVER; + } + + g_Dbs.db[i].superTblCount = stbSize; + for (int j = 0; j < stbSize; ++j) { + cJSON* stbInfo = cJSON_GetArrayItem(stables, j); + if (stbInfo == NULL) continue; + + // dbinfo + cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); + if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { + printf("failed to read json, stb name not found"); + goto PARSE_OVER; + } + strncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); + + cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); + if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { + printf("failed to read json, childtable_prefix not found"); + goto PARSE_OVER; + } + strncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); + + cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null + if (autoCreateTbl && autoCreateTbl->type == cJSON_String && autoCreateTbl->valuestring != NULL) { + if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; + } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + } else if (!autoCreateTbl) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + printf("failed to read json, auto_create_table not found"); + goto PARSE_OVER; + } + + cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no + if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) { + if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; + } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } + } else if (!childTblExists) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + printf("failed to read json, child_table_exists not found"); + goto PARSE_OVER; + } + + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); + if (!count || count->type != cJSON_Number || 0 >= count->valueint) { + printf("failed to read json, childtable_count not found"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; + + cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); + if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].dataSource, dataSource->valuestring, MAX_DB_NAME_SIZE); + } else if (!dataSource) { + strncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, data_source not found"); + goto PARSE_OVER; + } + + cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful + if (insertMode && insertMode->type == cJSON_String && insertMode->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].insertMode, insertMode->valuestring, MAX_DB_NAME_SIZE); + #ifndef TD_LOWA_CURL + if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 7)) { + printf("There no libcurl, so no support resetful test! please use taosc mode.\n"); + goto PARSE_OVER; + } + #endif + } else if (!insertMode) { + strncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, insert_mode not found"); + goto PARSE_OVER; + } + + cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); + if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].startTimestamp, ts->valuestring, MAX_DB_NAME_SIZE); + } else if (!ts) { + strncpy(g_Dbs.db[i].superTbls[j].startTimestamp, "now", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, start_timestamp not found"); + goto PARSE_OVER; + } + + cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); + if (timestampStep && timestampStep->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; + } else if (!timestampStep) { + g_Dbs.db[i].superTbls[j].timeStampStep = 1000; + } else { + printf("failed to read json, timestamp_step not found"); + goto PARSE_OVER; + } + + cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size"); + if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint; + if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } + } else if (!sampleDataBufSize) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } else { + printf("failed to read json, sample_buf_size not found"); + goto PARSE_OVER; + } + + cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); + if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].sampleFormat, sampleFormat->valuestring, MAX_DB_NAME_SIZE); + } else if (!sampleFormat) { + strncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, sample_format not found"); + goto PARSE_OVER; + } + + cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); + if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].sampleFile, sampleFile->valuestring, MAX_FILE_NAME_LEN); + } else if (!sampleFile) { + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, sample_file not found"); + goto PARSE_OVER; + } + + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); + if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { + strncpy(g_Dbs.db[i].superTbls[j].tagsFile, tagsFile->valuestring, MAX_FILE_NAME_LEN); + if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + g_Dbs.db[i].superTbls[j].tagSource = 1; + } + } else if (!tagsFile) { + memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + printf("failed to read json, tags_file not found"); + goto PARSE_OVER; + } + + cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + int32_t len = maxSqlLen->valueint; + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < TSDB_MAX_SQL_LEN) { + len = TSDB_MAX_SQL_LEN; + } + g_Dbs.db[i].superTbls[j].maxSqlLen = len; + } else if (!maxSqlLen) { + g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; + } else { + printf("failed to read json, maxSqlLen not found"); + goto PARSE_OVER; + } + + cJSON *multiThreadWriteOneTbl = cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes + if (multiThreadWriteOneTbl && multiThreadWriteOneTbl->type == cJSON_String && multiThreadWriteOneTbl->valuestring != NULL) { + if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; + } else { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } + } else if (!multiThreadWriteOneTbl) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } else { + printf("failed to read json, multiThreadWriteOneTbl not found"); + goto PARSE_OVER; + } + + cJSON* numberOfTblInOneSql = cJSON_GetObjectItem(stbInfo, "number_of_tbl_in_one_sql"); + if (numberOfTblInOneSql && numberOfTblInOneSql->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = numberOfTblInOneSql->valueint; + } else if (!numberOfTblInOneSql) { + g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = 0; + } else { + printf("failed to read json, numberOfTblInOneSql not found"); + goto PARSE_OVER; + } + + cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "rows_per_tbl"); + if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint; + } else if (!rowsPerTbl) { + g_Dbs.db[i].superTbls[j].rowsPerTbl = 1; + } else { + printf("failed to read json, rowsPerTbl not found"); + goto PARSE_OVER; + } + + cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); + if (disorderRatio && disorderRatio->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; + } else if (!disorderRatio) { + g_Dbs.db[i].superTbls[j].disorderRatio = 0; + } else { + printf("failed to read json, disorderRatio not found"); + goto PARSE_OVER; + } + + cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); + if (disorderRange && disorderRange->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; + } else if (!disorderRange) { + g_Dbs.db[i].superTbls[j].disorderRange = 1000; + } else { + printf("failed to read json, disorderRange not found"); + goto PARSE_OVER; + } + + cJSON* insertRate = cJSON_GetObjectItem(stbInfo, "insert_rate"); + if (insertRate && insertRate->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertRate = insertRate->valueint; + } else if (!insertRate) { + g_Dbs.db[i].superTbls[j].insertRate = 0; + } else { + printf("failed to read json, insert_rate not found"); + goto PARSE_OVER; + } + + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); + if (insertRows && insertRows->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; + if (0 == g_Dbs.db[i].superTbls[j].insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } + } else if (!insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } else { + printf("failed to read json, insert_rows not found"); + goto PARSE_OVER; + } + + if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + + int retVal = getColumnAndTagTypeFromInsertJsonFile(stbInfo, &g_Dbs.db[i].superTbls[j]); + if (false == retVal) { + goto PARSE_OVER; + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromQueryJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + strncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + strncpy(g_queryInfo.host, host->valuestring, MAX_DB_NAME_SIZE); + } else if (!host) { + strncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } else { + printf("failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_queryInfo.port = port->valueint; + } else if (!port) { + g_queryInfo.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + strncpy(g_queryInfo.user, user->valuestring, MAX_DB_NAME_SIZE); + } else if (!user) { + strncpy(g_queryInfo.user, "root", MAX_DB_NAME_SIZE); ; + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + strncpy(g_queryInfo.password, password->valuestring, MAX_DB_NAME_SIZE); + } else if (!password) { + strncpy(g_queryInfo.password, "taosdata", MAX_DB_NAME_SIZE);; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { + strncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); + } else if (!dbs) { + printf("failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); + if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { + strncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); + } else if (!queryMode) { + strncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); + } else { + printf("failed to read json, query_mode not found\n"); + goto PARSE_OVER; + } + + // super_table_query + cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query"); + if (!superQuery) { + g_queryInfo.superQueryInfo.concurrent = 0; + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superQuery->type != cJSON_Object) { + printf("failed to read json, super_table_query not found"); + goto PARSE_OVER; + } else { + cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval"); + if (rate && rate->type == cJSON_Number) { + g_queryInfo.superQueryInfo.rate = rate->valueint; + } else if (!rate) { + g_queryInfo.superQueryInfo.rate = 0; + } + + cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent"); + if (concurrent && concurrent->type == cJSON_Number) { + g_queryInfo.superQueryInfo.concurrent = concurrent->valueint; + } else if (!concurrent) { + g_queryInfo.superQueryInfo.concurrent = 1; + } + + cJSON* mode = cJSON_GetObjectItem(superQuery, "mode"); + if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { + if (0 == strcmp("sync", mode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", mode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 1; + } else { + printf("failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } + + cJSON* interval = cJSON_GetObjectItem(superQuery, "interval"); + if (interval && interval->type == cJSON_Number) { + g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint; + } else if (!interval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.subscribeInterval = 10000; + } + + cJSON* restart = cJSON_GetObjectItem(superQuery, "restart"); + if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { + if (0 == strcmp("yes", restart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", restart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 0; + } else { + printf("failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } + + cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (keepProgress && keepProgress->type == cJSON_String && keepProgress->valuestring != NULL) { + if (0 == strcmp("yes", keepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", keepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } else { + printf("failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!superSqls) { + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superSqls->type != cJSON_Array) { + printf("failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(superSqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.superQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(superSqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("failed to read json, sql not found\n"); + goto PARSE_OVER; + } + strncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { + strncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, super query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + // sub_table_query + cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query"); + if (!subQuery) { + g_queryInfo.subQueryInfo.threadCnt = 0; + g_queryInfo.subQueryInfo.sqlCount = 0; + } else if (subQuery->type != cJSON_Object) { + printf("failed to read json, sub_table_query not found"); + ret = true; + goto PARSE_OVER; + } else { + cJSON* subrate = cJSON_GetObjectItem(subQuery, "query_interval"); + if (subrate && subrate->type == cJSON_Number) { + g_queryInfo.subQueryInfo.rate = subrate->valueint; + } else if (!subrate) { + g_queryInfo.subQueryInfo.rate = 0; + } + + cJSON* threads = cJSON_GetObjectItem(subQuery, "threads"); + if (threads && threads->type == cJSON_Number) { + g_queryInfo.subQueryInfo.threadCnt = threads->valueint; + } else if (!threads) { + g_queryInfo.subQueryInfo.threadCnt = 1; + } + + //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count"); + //if (subTblCnt && subTblCnt->type == cJSON_Number) { + // g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint; + //} else if (!subTblCnt) { + // g_queryInfo.subQueryInfo.childTblCount = 0; + //} + + cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname"); + if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { + strncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); + } else { + printf("failed to read json, super table name not found\n"); + goto PARSE_OVER; + } + + cJSON* submode = cJSON_GetObjectItem(subQuery, "mode"); + if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { + if (0 == strcmp("sync", submode->valuestring)) { + g_queryInfo.subQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", submode->valuestring)) { + g_queryInfo.subQueryInfo.subscribeMode = 1; + } else { + printf("failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeMode = 0; + } + + cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval"); + if (subinterval && subinterval->type == cJSON_Number) { + g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint; + } else if (!subinterval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.subQueryInfo.subscribeInterval = 10000; + } + + cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart"); + if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { + if (0 == strcmp("yes", subrestart->valuestring)) { + g_queryInfo.subQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", subrestart->valuestring)) { + g_queryInfo.subQueryInfo.subscribeRestart = 0; + } else { + printf("failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeRestart = 1; + } + + cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress"); + if (subkeepProgress && subkeepProgress->type == cJSON_String && subkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", subkeepProgress->valuestring)) { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + } else { + printf("failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls"); + if (!subsqls) { + g_queryInfo.subQueryInfo.sqlCount = 0; + } else if (subsqls->type != cJSON_Array) { + printf("failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(subsqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.subQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(subsqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("failed to read json, sql not found\n"); + goto PARSE_OVER; + } + strncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ + strncpy(g_queryInfo.subQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("failed to read json, sub query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getInfoFromJsonFile(char* file) { + FILE *fp = fopen(file, "r"); + if (!fp) { + printf("failed to read %s, reason:%s\n", file, strerror(errno)); + return false; + } + + bool ret = false; + int maxLen = 64000; + char *content = calloc(1, maxLen + 1); + int len = fread(content, 1, maxLen, fp); + if (len <= 0) { + free(content); + fclose(fp); + printf("failed to read %s, content is null", file); + return false; + } + + content[len] = 0; + cJSON* root = cJSON_Parse(content); + if (root == NULL) { + printf("failed to cjson parse %s, invalid json format", file); + goto PARSE_OVER; + } + + cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); + if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { + if (0 == strcasecmp("insert", filetype->valuestring)) { + g_jsonType = INSERT_MODE; + } else if (0 == strcasecmp("query", filetype->valuestring)) { + g_jsonType = QUERY_MODE; + } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { + g_jsonType = SUBSCRIBE_MODE; + } else { + printf("failed to read json, filetype not support\n"); + goto PARSE_OVER; + } + } else if (!filetype) { + g_jsonType = INSERT_MODE; + } else { + printf("failed to read json, filetype not found\n"); + goto PARSE_OVER; + } + + if (INSERT_MODE == g_jsonType) { + ret = getMetaFromInsertJsonFile(root); + } else if (QUERY_MODE == g_jsonType) { + ret = getMetaFromQueryJsonFile(root); + } else if (SUBSCRIBE_MODE == g_jsonType) { + ret = getMetaFromQueryJsonFile(root); + } else { + printf("input json file type error! please input correct file type: insert or query or subscribe\n"); + goto PARSE_OVER; + } + +PARSE_OVER: + free(content); + cJSON_Delete(root); + fclose(fp); + return ret; +} + + +void prePareSampleData() { + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + //if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].dataSource, "sample", 6)) { + // readSampleFromFileToMem(&g_Dbs.db[i].superTbls[j]); + //} + + if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { + (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]); + } + + #ifdef TD_LOWA_CURL + if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 8)) { + curl_global_init(CURL_GLOBAL_ALL); + } + #endif + } + } +} + +void postFreeResource() { + tmfclose(g_fpOfInsertResult); + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (0 != g_Dbs.db[i].superTbls[j].colsOfCreatChildTable) { + free(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable); + g_Dbs.db[i].superTbls[j].colsOfCreatChildTable = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { + free(g_Dbs.db[i].superTbls[j].sampleDataBuf); + g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].childTblName) { + free(g_Dbs.db[i].superTbls[j].childTblName); + g_Dbs.db[i].superTbls[j].childTblName = NULL; + } + + #ifdef TD_LOWA_CURL + if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 8)) { + curl_global_cleanup(); + } + #endif + } + } +} + +int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* superTblInfo, int* sampleUsePos, FILE *fp, char* sampleBuf) { + if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) { + int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleBuf); + if (0 != ret) { + return -1; + } + *sampleUsePos = 0; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%s", sampleBuf + superTblInfo->lenOfOneRow * (*sampleUsePos)); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + + (*sampleUsePos)++; + + return dataLen; +} + +int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) { + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + for (int i = 0; i < stbInfo->columnCount; i++) { + if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { + if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { + printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); + return (-1); + } + + char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", stbInfo->columns[i].dataLen); + return (-1); + } + rand_string(buf, stbInfo->columns[i].dataLen); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "int", 3)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bigint", 6)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "float", 5)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "double", 6)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "smallint", 8)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + } else { + printf("No support data type: %s\n", stbInfo->columns[i].dataType); + return (-1); + } + } + dataLen -= 2; + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + + return dataLen; +} + +void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDataBuf) { + SSuperTable* superTblInfo = winfo->superTblInfo; + + int samplePos = 0; + + //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); + int64_t totalRowsInserted = 0; + int64_t totalAffectedRows = 0; + int64_t lastPrintTime = taosGetTimestampMs(); + + char* buffer = calloc(superTblInfo->maxSqlLen+1, 1); + if (NULL == buffer) { + printf("========calloc size[ %d ] fail!\n", superTblInfo->maxSqlLen); + return; + } + + int32_t numberOfTblInOneSql = superTblInfo->numberOfTblInOneSql; + int32_t tbls = winfo->end_table_id - winfo->start_table_id + 1; + if (numberOfTblInOneSql > tbls) { + numberOfTblInOneSql = tbls; + } + + int64_t time_counter = winfo->start_time; + int64_t tmp_time; + int sampleUsePos; + + int64_t st = 0; + int64_t et = 0; + for (int i = 0; i < superTblInfo->insertRows;) { + if (superTblInfo->insertRate && (et - st) < 1000) { + taosMsleep(1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + if (superTblInfo->insertRate) { + st = taosGetTimestampMs(); + } + + int32_t tbl_id = 0; + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { + int inserted = i; + + int k = 0; + int batchRowsSql = 0; + while (1) + { + int len = 0; + memset(buffer, 0, superTblInfo->maxSqlLen); + char *pstr = buffer; + + int32_t end_tbl_id = tID + numberOfTblInOneSql; + if (end_tbl_id > winfo->end_table_id) { + end_tbl_id = winfo->end_table_id+1; + } + for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) { + sampleUsePos = samplePos; + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample(superTblInfo, tbl_id % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + goto free_and_statistics; + } + + if (0 == len) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + } + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + if (0 == len) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); + } + } else { // pre-create child table + if (0 == len) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); + } + } + + tmp_time = time_counter; + for (k = 0; k < superTblInfo->rowsPerTbl;) { + int retLen = 0; + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); + if (retLen < 0) { + goto free_and_statistics; + } + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { + int64_t d = tmp_time - rand() % superTblInfo->disorderRange; + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); + } else { + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); + } + if (retLen < 0) { + goto free_and_statistics; + } + } + len += retLen; + //inserted++; + k++; + totalRowsInserted++; + batchRowsSql++; + + if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128) || batchRowsSql >= INT16_MAX - 1) { + tID = tbl_id + 1; + printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", superTblInfo->lenOfOneRow); + goto send_to_server; + } + } + + } + + tID = tbl_id; + inserted += superTblInfo->rowsPerTbl; + + send_to_server: + batchRowsSql = 0; + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + //printf("multi table===== sql: %s \n\n", buffer); + //int64_t t1 = taosGetTimestampMs(); + int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + if (0 > affectedRows) { + goto free_and_statistics; + } + totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + lastPrintTime = currentPrintTime; + } + //int64_t t2 = taosGetTimestampMs(); + //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); + } else { + #ifdef TD_LOWA_CURL + //int64_t t1 = taosGetTimestampMs(); + int retCode = curlProceSql(g_Dbs.host, g_Dbs.port, buffer, winfo->curl_handle); + //int64_t t2 = taosGetTimestampMs(); + //printf("http insert sql return, Spent %ld ms \n", t2 - t1); + + if (0 != retCode) { + printf("========curl return fail, threadID[%d]\n", winfo->threadID); + goto free_and_statistics; + } + #else + printf("========no use http mode for no curl lib!\n"); + goto free_and_statistics; + #endif + } + + //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); + break; + } + + if (tID > winfo->end_table_id) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + samplePos = sampleUsePos; + } + i = inserted; + time_counter = tmp_time; + } + } + + if (superTblInfo->insertRate) { + et = taosGetTimestampMs(); + } + //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); + } + + free_and_statistics: + tmfree(buffer); + winfo->totalRowsInserted = totalRowsInserted; + winfo->totalAffectedRows = totalAffectedRows; + printf("====thread[%d] completed total inserted rows: %"PRId64 ", affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + return; +} + +// sync insertion +/* + 1 thread: 100 tables * 2000 rows/s + 1 thread: 10 tables * 20000 rows/s + 6 thread: 300 tables * 2000 rows/s + + 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s +*/ +void *syncWrite(void *sarg) { + int64_t totalRowsInserted = 0; + int64_t totalAffectedRows = 0; + int64_t lastPrintTime = taosGetTimestampMs(); + + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + FILE *fp = NULL; + char* sampleDataBuf = NULL; + int samplePos = 0; + + // each thread read sample data from csv file + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + sampleDataBuf = calloc(superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); + if (sampleDataBuf == NULL) { + printf("Failed to calloc %d Bytes, reason:%s\n", superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); + return NULL; + } + + fp = fopen(superTblInfo->sampleFile, "r"); + if (fp == NULL) { + printf("Failed to open sample file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); + tmfree(sampleDataBuf); + return NULL; + } + int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleDataBuf); + if (0 != ret) { + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; + } + } + + if (superTblInfo->numberOfTblInOneSql > 0) { + syncWriteForNumberOfTblInOneSql(winfo, fp, sampleDataBuf); + tmfree(sampleDataBuf); + tmfclose(fp); + return NULL; + } + + //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); + + char* buffer = calloc(superTblInfo->maxSqlLen, 1); + + int nrecords_per_request = 0; + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + nrecords_per_request = (superTblInfo->maxSqlLen - 1280 - superTblInfo->lenOfTagOfOneRow) / superTblInfo->lenOfOneRow; + } else { + nrecords_per_request = (superTblInfo->maxSqlLen - 1280) / superTblInfo->lenOfOneRow; + } + + int nrecords_no_last_req = nrecords_per_request; + int nrecords_last_req = 0; + int loop_cnt = 0; + if (0 != superTblInfo->insertRate) { + if (nrecords_no_last_req >= superTblInfo->insertRate) { + nrecords_no_last_req = superTblInfo->insertRate; + } else { + nrecords_last_req = superTblInfo->insertRate % nrecords_per_request; + loop_cnt = (superTblInfo->insertRate / nrecords_per_request) + (superTblInfo->insertRate % nrecords_per_request ? 1 : 0) ; + } + } + + if (nrecords_no_last_req <= 0) { + nrecords_no_last_req = 1; + } + + if (nrecords_no_last_req >= INT16_MAX) { + nrecords_no_last_req = INT16_MAX - 1; + } + + if (nrecords_last_req >= INT16_MAX) { + nrecords_last_req = INT16_MAX - 1; + } + + int nrecords_cur_req = nrecords_no_last_req; + int loop_cnt_orig = loop_cnt; + + //printf("========nrecords_per_request:%d, nrecords_no_last_req:%d, nrecords_last_req:%d, loop_cnt:%d\n", nrecords_per_request, nrecords_no_last_req, nrecords_last_req, loop_cnt); + + int64_t time_counter = winfo->start_time; + + int64_t st = 0; + int64_t et = 0; + for (int i = 0; i < superTblInfo->insertRows;) { + if (superTblInfo->insertRate && (et - st) < 1000) { + taosMsleep(1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + if (superTblInfo->insertRate) { + st = taosGetTimestampMs(); + } + + for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { + int inserted = i; + int64_t tmp_time = time_counter; + + int sampleUsePos = samplePos; + int k = 0; + while (1) + { + int len = 0; + memset(buffer, 0, superTblInfo->maxSqlLen); + char *pstr = buffer; + + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo); + } else { + tagsValBuf = getTagValueFromTagSample(superTblInfo, tID % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + goto free_and_statistics_2; + } + + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values", winfo->db_name, superTblInfo->childTblPrefix, tID, winfo->db_name, superTblInfo->sTblName, tagsValBuf); + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values", winfo->db_name, superTblInfo->childTblName + tID * TSDB_TABLE_NAME_LEN); + } else { + len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values", winfo->db_name, superTblInfo->childTblPrefix, tID); + } + + for (k = 0; k < nrecords_cur_req;) { + int retLen = 0; + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); + if (retLen < 0) { + goto free_and_statistics_2; + } + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { + int64_t d = tmp_time - rand() % superTblInfo->disorderRange; + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); + //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); + } else { + retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); + } + if (retLen < 0) { + goto free_and_statistics_2; + } + } + len += retLen; + inserted++; + k++; + totalRowsInserted++; + + if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) break; + } + + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + //printf("===== sql: %s \n\n", buffer); + //int64_t t1 = taosGetTimestampMs(); + int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); + if (0 > affectedRows){ + goto free_and_statistics_2; + } + totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + lastPrintTime = currentPrintTime; + } + //int64_t t2 = taosGetTimestampMs(); + //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); + } else { + #ifdef TD_LOWA_CURL + //int64_t t1 = taosGetTimestampMs(); + int retCode = curlProceSql(g_Dbs.host, g_Dbs.port, buffer, winfo->curl_handle); + //int64_t t2 = taosGetTimestampMs(); + //printf("http insert sql return, Spent %ld ms \n", t2 - t1); + + if (0 != retCode) { + printf("========curl return fail, threadID[%d]\n", winfo->threadID); + goto free_and_statistics_2; + } + #else + printf("========no use http mode for no curl lib!\n"); + goto free_and_statistics_2; + #endif + } + + //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); + + if (loop_cnt) { + loop_cnt--; + if ((1 == loop_cnt) && (0 != nrecords_last_req)) { + nrecords_cur_req = nrecords_last_req; + } else if (0 == loop_cnt){ + nrecords_cur_req = nrecords_no_last_req; + loop_cnt = loop_cnt_orig; + break; + } + } else { + break; + } + } + + if (tID == winfo->end_table_id) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { + samplePos = sampleUsePos; + } + i = inserted; + time_counter = tmp_time; + } + } + + if (superTblInfo->insertRate) { + et = taosGetTimestampMs(); + } + //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); + } + + free_and_statistics_2: + tmfree(buffer); + tmfree(sampleDataBuf); + tmfclose(fp); + + winfo->totalRowsInserted = totalRowsInserted; + winfo->totalAffectedRows = totalAffectedRows; + + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); + return NULL; +} + +void callBack(void *param, TAOS_RES *res, int code) { + threadInfo* winfo = (threadInfo*)param; + + if (winfo->superTblInfo->insertRate) { + winfo->et = taosGetTimestampMs(); + if (winfo->et - winfo->st < 1000) { + taosMsleep(1000 - (winfo->et - winfo->st)); // ms + } + } + + char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); + char *data = calloc(1, MAX_DATA_SIZE); + char *pstr = buffer; + pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id); + if (winfo->counter >= winfo->superTblInfo->insertRows) { + winfo->start_table_id++; + winfo->counter = 0; + } + if (winfo->start_table_id > winfo->end_table_id) { + tsem_post(&winfo->lock_sem); + free(buffer); + free(data); + taos_free_result(res); + return; + } + + for (int i = 0; i < winfo->nrecords_per_request; i++) { + int rand_num = rand() % 100; + if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) + { + int64_t d = winfo->lastTs - rand() % 1000000 + rand_num; + //generateData(data, datatype, ncols_per_record, d, len_of_binary); + (void)generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo); + } else { + //generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); + (void)generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo); + } + pstr += sprintf(pstr, "%s", data); + winfo->counter++; + + if (winfo->counter >= winfo->superTblInfo->insertRows) { + break; + } + } + + if (winfo->superTblInfo->insertRate) { + winfo->st = taosGetTimestampMs(); + } + taos_query_a(winfo->taos, buffer, callBack, winfo); + free(buffer); + free(data); + + taos_free_result(res); +} + +void *asyncWrite(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + winfo->nrecords_per_request = 0; + //if (AUTO_CREATE_SUBTBL == winfo->superTblInfo->autoCreateTable) { + winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280 - winfo->superTblInfo->lenOfTagOfOneRow) / winfo->superTblInfo->lenOfOneRow; + //} else { + // winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280) / winfo->superTblInfo->lenOfOneRow; + //} + + if (0 != winfo->superTblInfo->insertRate) { + if (winfo->nrecords_per_request >= winfo->superTblInfo->insertRate) { + winfo->nrecords_per_request = winfo->superTblInfo->insertRate; + } + } + + if (winfo->nrecords_per_request <= 0) { + winfo->nrecords_per_request = 1; + } + + if (winfo->nrecords_per_request >= INT16_MAX) { + winfo->nrecords_per_request = INT16_MAX - 1; + } + + if (winfo->nrecords_per_request >= INT16_MAX) { + winfo->nrecords_per_request = INT16_MAX - 1; + } + + winfo->st = 0; + winfo->et = 0; + winfo->lastTs = winfo->start_time; + + if (winfo->superTblInfo->insertRate) { + winfo->st = taosGetTimestampMs(); + } + taos_query_a(winfo->taos, "show databases", callBack, winfo); + + tsem_wait(&(winfo->lock_sem)); + + return NULL; +} + +void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* superTblInfo) { + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); + int ntables = superTblInfo->childTblCount; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + TAOS* taos; + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + if (NULL == taos) { + printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); + exit(-1); + } + } + + int32_t timePrec = TSDB_TIME_PRECISION_MILLI; + if (0 != precision[0]) { + if (0 == strncasecmp(precision, "ms", 2)) { + timePrec = TSDB_TIME_PRECISION_MILLI; + } else if (0 == strncasecmp(precision, "us", 2)) { + timePrec = TSDB_TIME_PRECISION_MICRO; + } else { + printf("No support precision: %s\n", precision); + exit(-1); + } + } + + int64_t start_time; + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + start_time = taosGetTimestamp(timePrec); + } else { + (void)taosParseTime(superTblInfo->startTimestamp, &start_time, strlen(superTblInfo->startTimestamp), timePrec, 0); + } + + double start = getCurrentTime(); + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->superTblInfo = superTblInfo; + + t_info->start_time = start_time; + + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + t_info->taos = taos; + } else { + t_info->taos = NULL; + #ifdef TD_LOWA_CURL + t_info->curl_handle = curl_easy_init(); + #endif + } + + if (0 == superTblInfo->multiThreadWriteOneTbl) { + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + last = t_info->end_table_id + 1; + } else { + t_info->start_table_id = 0; + t_info->end_table_id = superTblInfo->childTblCount - 1; + t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); + } + + tsem_init(&(t_info->lock_sem), 0, 0); + + if (SYNC == g_Dbs.queryMode) { + pthread_create(pids + i, NULL, syncWrite, t_info); + } else { + pthread_create(pids + i, NULL, asyncWrite, t_info); + } + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + + tsem_destroy(&(t_info->lock_sem)); + + superTblInfo->totalAffectedRows += t_info->totalAffectedRows; + superTblInfo->totalRowsInserted += t_info->totalRowsInserted; + #ifdef TD_LOWA_CURL + if (t_info->curl_handle) { + curl_easy_cleanup(t_info->curl_handle); + } + #endif + } + + double end = getCurrentTime(); + + taos_close(taos); + + free(pids); + free(infos); + + printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s\n\n", + end - start, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName); + fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s\n\n", + end - start, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName); +} + + +void *readTable(void *sarg) { +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; + TAOS *taos = rinfo->taos; + char command[BUFFER_SIZE] = "\0"; + int64_t sTime = rinfo->start_time; + char *tb_prefix = rinfo->tb_prefix; + FILE *fp = fopen(rinfo->fp, "a"); + if (NULL == fp) { + printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + return NULL; + } + + int num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; + int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; + int totalData = num_of_DPT * num_of_tables; + bool do_aggreFunc = g_Dbs.do_aggreFunc; + + int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + if (!do_aggreFunc) { + printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); + } + printf("%d records:\n", totalData); + fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); + + for (int j = 0; j < n; j++) { + double totalT = 0; + int count = 0; + for (int i = 0; i < num_of_tables; i++) { + sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime); + + double t = getCurrentTime(); + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + return NULL; + } + + while (taos_fetch_row(pSql) != NULL) { + count++; + } + + t = getCurrentTime() - t; + totalT += t; + + taos_free_result(pSql); + } + + fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", + aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, + (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); + printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT); + } + fprintf(fp, "\n"); + fclose(fp); +#endif + return NULL; +} + +void *readMetric(void *sarg) { +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; + TAOS *taos = rinfo->taos; + char command[BUFFER_SIZE] = "\0"; + FILE *fp = fopen(rinfo->fp, "a"); + if (NULL == fp) { + printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + return NULL; + } + + int num_of_DPT = rinfo->superTblInfo->insertRows; + int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; + int totalData = num_of_DPT * num_of_tables; + bool do_aggreFunc = g_Dbs.do_aggreFunc; + + int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; + if (!do_aggreFunc) { + printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); + } + printf("%d records:\n", totalData); + fprintf(fp, "Querying On %d records:\n", totalData); + + for (int j = 0; j < n; j++) { + char condition[BUFFER_SIZE - 30] = "\0"; + char tempS[64] = "\0"; + + int m = 10 < num_of_tables ? 10 : num_of_tables; + + for (int i = 1; i <= m; i++) { + if (i == 1) { + sprintf(tempS, "t1 = %d", i); + } else { + sprintf(tempS, " or t1 = %d ", i); + } + strcat(condition, tempS); + + sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); + + printf("Where condition: %s\n", condition); + fprintf(fp, "%s\n", command); + + double t = getCurrentTime(); + + TAOS_RES *pSql = taos_query(taos, command); + int32_t code = taos_errno(pSql); + + if (code != 0) { + fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + return NULL; + } + int count = 0; + while (taos_fetch_row(pSql) != NULL) { + count++; + } + t = getCurrentTime() - t; + + fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / t, t * 1000); + printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t); + + taos_free_result(pSql); + } + fprintf(fp, "\n"); + } + fclose(fp); +#endif + return NULL; +} + + +int insertTestProcess() { + + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); + if (NULL == g_fpOfInsertResult) { + fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); + return 1; + }; + + printfInsertMeta(); + printfInsertMetaToFile(g_fpOfInsertResult); + + printf("Press enter key to continue\n\n"); + (void)getchar(); + + init_rand_data(); + + // create database and super tables + (void)createDatabases(); + + // pretreatement + prePareSampleData(); + + double start; + double end; + + // create child tables + start = getCurrentTime(); + createChildTables(); + end = getCurrentTime(); + if (g_totalChildTables > 0) { + printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); + fprintf(g_fpOfInsertResult, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); + } + + usleep(1000*1000); + + // create sub threads for inserting data + //start = getCurrentTime(); + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + startMultiThreadInsertData(g_Dbs.threadCount, g_Dbs.db[i].dbName, g_Dbs.db[i].dbCfg.precision, superTblInfo); + } + } + //end = getCurrentTime(); + + //int64_t totalRowsInserted = 0; + //int64_t totalAffectedRows = 0; + //for (int i = 0; i < g_Dbs.dbCount; i++) { + // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // totalRowsInserted += g_Dbs.db[i].superTbls[j].totalRowsInserted; + // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; + //} + //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalRowsInserted, totalAffectedRows, g_Dbs.threadCount); + if (NULL == g_args.metaFile && false == g_Dbs.insert_only) { + // query data + pthread_t read_id; + threadInfo *rInfo = malloc(sizeof(threadInfo)); + rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + rInfo->start_table_id = 0; + rInfo->end_table_id = g_Dbs.db[0].superTbls[0].childTblCount - 1; + //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + //rInfo->nrecords_per_table = g_Dbs.db[0].superTbls[0].insertRows; + rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + rInfo->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); + strcpy(rInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix); + strcpy(rInfo->fp, g_Dbs.resultFile); + + if (!g_Dbs.use_metric) { + pthread_create(&read_id, NULL, readTable, rInfo); + } else { + pthread_create(&read_id, NULL, readMetric, rInfo); + } + pthread_join(read_id, NULL); + taos_close(rInfo->taos); + } + + postFreeResource(); + + return 0; +} + +void *superQueryProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + //char sqlStr[MAX_TB_NAME_SIZE*2]; + //sprintf(sqlStr, "use %s", g_queryInfo.dbName); + //queryDB(winfo->taos, sqlStr); + + int64_t st = 0; + int64_t et = 0; + while (1) { + if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + st = taosGetTimestampMs(); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + int64_t t1 = taosGetTimestampUs(); + selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], g_queryInfo.superQueryInfo.result[i]); + int64_t t2 = taosGetTimestampUs(); + printf("taosc select sql return, Spent %f s\n", (t2 - t1)/1000000.0); + } else { + #ifdef TD_LOWA_CURL + int64_t t1 = taosGetTimestampUs(); + int retCode = curlProceSql(g_queryInfo.host, g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i], winfo->curl_handle); + int64_t t2 = taosGetTimestampUs(); + printf("http select sql return, Spent %f s \n", (t2 - t1)/1000000.0); + + if (0 != retCode) { + printf("========curl return fail, threadID[%d]\n", winfo->threadID); + return NULL; + } + #endif + } + } + et = taosGetTimestampMs(); + printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.6fs\n\n", (uint64_t)pthread_self(), (double)(et - st)/1000.0); + } + return NULL; +} + +void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { + char sourceString[32] = "xxxx"; + char subTblName[MAX_TB_NAME_SIZE*3]; + sprintf(subTblName, "%s.%s", g_queryInfo.dbName, g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); + + //printf("inSql: %s\n", inSql); + + char* pos = strstr(inSql, sourceString); + if (0 == pos) { + return; + } + + strncpy(outSql, inSql, pos - inSql); + //printf("1: %s\n", outSql); + strcat(outSql, subTblName); + //printf("2: %s\n", outSql); + strcat(outSql, pos+strlen(sourceString)); + //printf("3: %s\n", outSql); +} + +void *subQueryProcess(void *sarg) { + char sqlstr[1024]; + threadInfo *winfo = (threadInfo *)sarg; + int64_t st = 0; + int64_t et = 0; + while (1) { + if (g_queryInfo.subQueryInfo.rate && (et - st) < g_queryInfo.subQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.subQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + } + + st = taosGetTimestampMs(); + for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + memset(sqlstr,0,sizeof(sqlstr)); + replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], sqlstr, i); + selectAndGetResult(winfo->taos, sqlstr, g_queryInfo.subQueryInfo.result[i]); + } + } + et = taosGetTimestampMs(); + printf("========thread[%"PRIu64"] complete all sqls to allocate all sub-tables once queries duration:%.4fs\n\n", (uint64_t)pthread_self(), (double)(et - st)/1000.0); + } + return NULL; +} + +int queryTestProcess() { + printfQueryMeta(); + + printf("Press enter key to continue\n\n"); + (void)getchar(); + + TAOS * taos = NULL; + taos_init(); + taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.subQueryInfo.sqlCount) { + (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); + } + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from super table + if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { + + pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + t_info->taos = taos; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + (void)queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE); + } else { + t_info->taos = NULL; + #ifdef TD_LOWA_CURL + t_info->curl_handle = curl_easy_init(); + #endif + } + + pthread_create(pids + i, NULL, superQueryProcess, t_info); + } + }else { + g_queryInfo.superQueryInfo.concurrent = 0; + } + + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + //==== create sub threads for query from sub table + if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + int ntables = g_queryInfo.subQueryInfo.childTblCount; + int threads = g_queryInfo.subQueryInfo.threadCnt; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + t_info->taos = taos; + pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); + } + + g_queryInfo.subQueryInfo.threadCnt = threads; + }else { + g_queryInfo.subQueryInfo.threadCnt = 0; + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); + } + + tmfree((char*)pids); + tmfree((char*)infos); + + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); + + taos_close(taos); + return 0; +} + +static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + printf("failed to subscribe result, code:%d, reason:%s\n", code, taos_errstr(res)); + return; + } + + getResult(res, (char*)param); + taos_free_result(res); +} + +static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { + TAOS_SUB* tsub = NULL; + + if (g_queryInfo.superQueryInfo.subscribeMode) { + tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.superQueryInfo.subscribeInterval); + } else { + tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, NULL, NULL, 0); + } + + if (tsub == NULL) { + printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql); + return NULL; + } + + return tsub; +} + +void *subSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + char subSqlstr[1024]; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){ + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + memset(subSqlstr,0,sizeof(subSqlstr)); + replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i); + g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, g_queryInfo.subQueryInfo.result[i]); + if (NULL == g_queryInfo.subQueryInfo.tsub[i]) { + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", pthread_self(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + while (1) { + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.subQueryInfo.subscribeMode) { + continue; + } + + TAOS_RES* res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]); + if (res) { + getResult(res, g_queryInfo.subQueryInfo.result[i]); + taos_free_result(res); + } + } + } + + for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], g_queryInfo.subQueryInfo.subscribeKeepProgress); + } + return NULL; +} + +void *superSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) { + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.superQueryInfo.sql[i], topic, g_queryInfo.superQueryInfo.result[i]); + if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", pthread_self(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + while (1) { + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.superQueryInfo.subscribeMode) { + continue; + } + + TAOS_RES* res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); + if (res) { + getResult(res, g_queryInfo.superQueryInfo.result[i]); + taos_free_result(res); + } + } + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + return NULL; +} + +int subscribeTestProcess() { + printfQueryMeta(); + + printf("Press enter key to continue\n\n"); + (void)getchar(); + + TAOS * taos = NULL; + taos_init(); + taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.subQueryInfo.sqlCount) { + (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); + } + + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from super table + if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { + pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + t_info->taos = taos; + pthread_create(pids + i, NULL, superSubscribeProcess, t_info); + } + } + + //==== create sub threads for query from sub table + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + printf("malloc failed for create threads\n"); + taos_close(taos); + exit(-1); + } + + int ntables = g_queryInfo.subQueryInfo.childTblCount; + int threads = g_queryInfo.subQueryInfo.threadCnt; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + int last = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_id = last; + t_info->end_table_id = i < b ? last + a : last + a - 1; + t_info->taos = taos; + pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); + } + g_queryInfo.subQueryInfo.threadCnt = threads; + } + + for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); + } + + tmfree((char*)pids); + tmfree((char*)infos); + + for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); + taos_close(taos); + return 0; +} + +void initOfInsertMeta() { + memset(&g_Dbs, 0, sizeof(SDbs)); + + // set default values + strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_Dbs.port = 6030; + strncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + strncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); + g_Dbs.threadCount = 2; + g_Dbs.use_metric = true; +} + +void initOfQueryMeta() { + memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); + + // set default values + strncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); + g_queryInfo.port = 6030; + strncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); + strncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); +} + +void setParaFromArg(){ + if (g_args.host) { + strcpy(g_Dbs.host, g_args.host); + } else { + strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); + } + + if (g_args.user) { + strcpy(g_Dbs.user, g_args.user); + } + + if (g_args.password) { + strcpy(g_Dbs.password, g_args.password); + } + + if (g_args.port) { + g_Dbs.port = g_args.port; + } + + g_Dbs.dbCount = 1; + g_Dbs.db[0].drop = 1; + + strncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); + g_Dbs.db[0].dbCfg.replica = g_args.replica; + strncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + + + strncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); + + g_Dbs.use_metric = g_args.use_metric; + g_Dbs.insert_only = g_args.insert_only; + + g_Dbs.db[0].superTblCount = 1; + strncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.queryMode = g_args.mode; + + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].insertRate = 0; + g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; + g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; + strncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + strncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + strncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); + strncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].timeStampStep = 10; + + // g_args.num_of_RPR; + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; + + g_Dbs.do_aggreFunc = true; + + char dataString[STRING_LEN]; + char **data_type = g_args.datatype; + + memset(dataString, 0, STRING_LEN); + + if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 || strcasecmp(data_type[0], "NCHAR") == 0 ) { + g_Dbs.do_aggreFunc = false; + } + + g_Dbs.db[0].superTbls[0].columnCount = 0; + for (int i = 0; i < MAX_NUM_DATATYPE; i++) { + if (data_type[i] == NULL) { + break; + } + + strncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, data_type[i], MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columnCount++; + } + + if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { + g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; + } else { + for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { + strncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount++; + } + } + + if (g_Dbs.use_metric) { + strncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; + + strncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tagCount = 2; + } else { + g_Dbs.db[0].superTbls[0].tagCount = 0; + } +} + +/* Function to do regular expression check */ +static int regexMatch(const char *s, const char *reg, int cflags) { + regex_t regex; + char msgbuf[100] = {0}; + + /* Compile regular expression */ + if (regcomp(®ex, reg, cflags) != 0) { + printf("Fail to compile regex\n"); + exit(-1); + } + + /* Execute regular expression */ + int reti = regexec(®ex, s, 0, NULL, 0); + if (!reti) { + regfree(®ex); + return 1; + } else if (reti == REG_NOMATCH) { + regfree(®ex); + return 0; + } else { + regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); + printf("Regex match failed: %s\n", msgbuf); + regfree(®ex); + exit(-1); + } + + return 0; +} + +static int isCommentLine(char *line) { + if (line == NULL) return 1; + + return regexMatch(line, "^\\s*#.*", REG_EXTENDED); +} + +void querySqlFile(TAOS* taos, char* sqlFile) +{ + FILE *fp = fopen(sqlFile, "r"); + if (fp == NULL) { + printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); + return; + } + + int read_len = 0; + char * cmd = calloc(1, MAX_SQL_SIZE); + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; + + double t = getCurrentTime(); + + while ((read_len = tgetline(&line, &line_len, fp)) != -1) { + if (read_len >= MAX_SQL_SIZE) continue; + line[--read_len] = '\0'; + + if (read_len == 0 || isCommentLine(line)) { // line starts with # + continue; + } + + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } + + memcpy(cmd + cmd_len, line, read_len); + queryDbExec(taos, cmd, NO_INSERT_TYPE); + memset(cmd, 0, MAX_SQL_SIZE); + cmd_len = 0; + } + + t = getCurrentTime() - t; + printf("run %s took %.6f second(s)\n\n", sqlFile, t); + + tmfree(cmd); + tmfree(line); + tmfclose(fp); + return; +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv, &g_args); + + if (g_args.metaFile) { + initOfInsertMeta(); + initOfQueryMeta(); + if (false == getInfoFromJsonFile(g_args.metaFile)) { + printf("Failed to read %s\n", g_args.metaFile); + return 1; + } + } else { + + memset(&g_Dbs, 0, sizeof(SDbs)); + g_jsonType = INSERT_MODE; + setParaFromArg(); + + if (NULL != g_args.sqlFile) { + TAOS* qtaos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); + querySqlFile(qtaos, g_args.sqlFile); + taos_close(qtaos); + return 0; + } + + (void)insertTestProcess(); + if (g_Dbs.insert_only) return 0; + + // select + + //printf("At present, there is no integration of taosdemo, please wait patiently!\n"); + return 0; + } + + if (INSERT_MODE == g_jsonType) { + if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); + (void)insertTestProcess(); + } else if (QUERY_MODE == g_jsonType) { + if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + (void)queryTestProcess(); + } else if (SUBSCRIBE_MODE == g_jsonType) { + if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); + (void)subscribeTestProcess(); + } else { + ; + } + + taos_cleanup(); + return 0; +} + diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 588d21574b6d4c07b746ee487b0449c705186298..60707f22e297e3cd253d88cc9bed962dd36dacd1 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -332,6 +332,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'N': arguments->data_batch = atoi(arg); + if (arguments->data_batch >= INT16_MAX) { + arguments->data_batch = INT16_MAX - 1; + } break; case 'L': { diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 583acc19b4b9f75fe82e676e705bf1e8d0a0ecd9..a07607e615a46c34c1a570cacc42c8f36769b754 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -128,8 +128,8 @@ typedef struct { typedef struct { int32_t dnodeId; int8_t role; - int8_t reserved[3]; - SDnodeObj* pDnode; + int8_t vver[3]; // To ensure compatibility, 3 bits are used to represent the remainder of 64 bit version + SDnodeObj *pDnode; } SVnodeGid; typedef struct SVgObj { @@ -174,7 +174,8 @@ typedef struct { int8_t replications; int8_t quorum; int8_t update; - int8_t reserved[11]; + int8_t cacheLastRow; + int8_t reserved[10]; } SDbCfg; typedef struct SDbObj { diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index b959da73e8144f58de6dfba144ce49bd2880b5a1..56d7455ad24f76a0b289e8ad8882db1867d0bed9 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -52,6 +52,9 @@ typedef enum EDnodeOfflineReason { TAOS_DN_OFF_TIME_ZONE_NOT_MATCH, TAOS_DN_OFF_LOCALE_NOT_MATCH, TAOS_DN_OFF_CHARSET_NOT_MATCH, + TAOS_DN_OFF_FLOW_CTRL_NOT_MATCH, + TAOS_DN_OFF_SLAVE_QUERY_NOT_MATCH, + TAOS_DN_OFF_ADJUST_MASTER_NOT_MATCH, TAOS_DN_OFF_OTHERS } EDnodeOfflineReason; diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index ee9ec7ae9305c3d436b6e5bec0cd4e94ecc1adf4..2067ad04ccf963701bffcf134c8ae478547f1abb 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -53,6 +53,9 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromIp(char *ep); +int32_t mnodeGetVgidVer(int8_t *vver); +void mnodeSetVgidVer(int8_t *cver, uint64_t iver); + #ifdef __cplusplus } #endif diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 56229daffa87d3300cc597d9353b5a720751a0f9..a35e30481077d0e16888d0ee036312cd068ea9b4 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -171,7 +171,7 @@ void mnodeUpdateClusterId() { void *pIter = mnodeGetNextCluster(NULL, &pCluster); if (pCluster != NULL) { tstrncpy(tsClusterId, pCluster->uid, TSDB_CLUSTER_ID_LEN); - mInfo("cluster id is set to %s", tsClusterId); + mDebug("cluster id is set to %s", tsClusterId); } mnodeDecClusterRef(pCluster); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 0eb98289597ef1bbcb6e6bf357c1f41796bbe580..333844ccbea801f6ad0bd3c31bdf0ee7f9287c85 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -322,6 +322,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { return TSDB_CODE_MND_INVALID_DB_OPTION; } + if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { + mError("invalid db option cacheLastRow:%d valid range: [%d, %d]", pCfg->cacheLastRow, TSDB_MIN_DB_CACHE_LAST_ROW, TSDB_MAX_DB_CACHE_LAST_ROW); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + return TSDB_CODE_SUCCESS; } @@ -343,6 +348,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->replications < 0) pCfg->replications = tsReplications; if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; if (pCfg->update < 0) pCfg->update = tsUpdate; + if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; } static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { @@ -396,7 +402,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * .walLevel = pCreate->walLevel, .replications = pCreate->replications, .quorum = pCreate->quorum, - .update = pCreate->update + .update = pCreate->update, + .cacheLastRow = pCreate->cacheLastRow }; mnodeSetDefaultDbCfg(&pDb->cfg); @@ -605,6 +612,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn strcpy(pSchema[cols].name, "comp"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "cachelast"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; #ifndef __CLOUD_VERSION__ } #endif @@ -750,6 +763,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int8_t *)pWrite = pDb->cfg.compression; cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.cacheLastRow; + cols++; #ifndef __CLOUD_VERSION__ } #endif @@ -864,6 +881,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { int8_t quorum = pAlter->quorum; int8_t precision = pAlter->precision; int8_t update = pAlter->update; + int8_t cacheLastRow = pAlter->cacheLastRow; terrno = TSDB_CODE_SUCCESS; @@ -976,6 +994,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { #endif } + if (cacheLastRow >= 0 && cacheLastRow != pDb->cfg.cacheLastRow) { + mDebug("db:%s, cacheLastRow:%d change to %d", pDb->name, pDb->cfg.cacheLastRow, cacheLastRow); + newCfg.cacheLastRow = cacheLastRow; + } + return newCfg; } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 27588669ebafc856eb0284cf0bf1f6ee12b6b3f9..14d1fa5816ffc168463e0392e675e20cc783af7c 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -375,10 +375,6 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { mError("\"numOfMnodes\"[%d - %d] cfg parameters inconsistent", clusterCfg->numOfMnodes, htonl(tsNumOfMnodes)); return TAOS_DN_OFF_NUM_OF_MNODES_NOT_MATCH; } - if (clusterCfg->enableBalance != htonl(tsEnableBalance)) { - mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, htonl(tsEnableBalance)); - return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH; - } if (clusterCfg->mnodeEqualVnodeNum != htonl(tsMnodeEqualVnodeNum)) { mError("\"mnodeEqualVnodeNum\"[%d - %d] cfg parameters inconsistent", clusterCfg->mnodeEqualVnodeNum, htonl(tsMnodeEqualVnodeNum)); @@ -428,6 +424,23 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { return TAOS_DN_OFF_CHARSET_NOT_MATCH; } + if (clusterCfg->enableBalance != tsEnableBalance) { + mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, tsEnableBalance); + return TAOS_DN_OFF_ENABLE_BALANCE_NOT_MATCH; + } + if (clusterCfg->flowCtrl != tsEnableFlowCtrl) { + mError("\"flowCtrl\"[%d - %d] cfg parameters inconsistent", clusterCfg->flowCtrl, tsEnableFlowCtrl); + return TAOS_DN_OFF_FLOW_CTRL_NOT_MATCH; + } + if (clusterCfg->slaveQuery != tsEnableSlaveQuery) { + mError("\"slaveQuery\"[%d - %d] cfg parameters inconsistent", clusterCfg->slaveQuery, tsEnableSlaveQuery); + return TAOS_DN_OFF_SLAVE_QUERY_NOT_MATCH; + } + if (clusterCfg->adjustMaster != tsEnableAdjustMaster) { + mError("\"adjustMaster\"[%d - %d] cfg parameters inconsistent", clusterCfg->adjustMaster, tsEnableAdjustMaster); + return TAOS_DN_OFF_ADJUST_MASTER_NOT_MATCH; + } + return 0; } @@ -558,6 +571,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pVload->vgId = htonl(pVload->vgId); pVload->dbCfgVersion = htonl(pVload->dbCfgVersion); pVload->vgCfgVersion = htonl(pVload->vgCfgVersion); + pVload->vnodeVersion = htobe64(pVload->vnodeVersion); SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId); if (pVgroup == NULL) { @@ -1031,6 +1045,11 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int8_t *)cfg->ptr)); + varDataSetLen(pWrite, t); + numOfRows++; + break; case TAOS_CFG_VTYPE_INT16: t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); varDataSetLen(pWrite, t); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 8b3b2896fffa3e767028f87c52b36f21331c2234..3ea41c41c6ffe89671853387e59733f7db7506cd 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -377,6 +377,25 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) { return code; } +static bool mnodeAllOnline() { + void *pIter = NULL; + bool allOnline = true; + + while (1) { + SMnodeObj *pMnode = NULL; + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + if (pMnode->role != TAOS_SYNC_ROLE_MASTER && pMnode->role != TAOS_SYNC_ROLE_SLAVE) { + allOnline = false; + mDebug("mnode:%d, role:%s, not online", pMnode->mnodeId, syncRole[pMnode->role]); + mnodeDecMnodeRef(pMnode); + } + } + mnodeCancelGetNextMnode(pIter); + + return allOnline; +} + void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj)); pMnode->mnodeId = dnodeId; @@ -389,6 +408,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { .fpRsp = mnodeCreateMnodeCb }; + if (needConfirm && !mnodeAllOnline()) { + mDebug("wait all mnode online then create new mnode"); + return; + } + int32_t code = TSDB_CODE_SUCCESS; if (needConfirm) { code = mnodeSendCreateMnodeMsg(dnodeId, dnodeEp); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1ab6a363e77dfbe2a3272712c37190e6ebc279d5..6997d0a666555e1237b7f5e019368c822ed2a6b1 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -225,6 +225,10 @@ void sdbUpdateMnodeRoles() { for (int32_t i = 0; i < tsSdbMgmt.cfg.replica; ++i) { SMnodeObj *pMnode = mnodeGetMnode(roles.nodeId[i]); if (pMnode != NULL) { + if (pMnode->role != roles.role[i]) { + bnNotify(); + } + pMnode->role = roles.role[i]; sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]); if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role; @@ -1081,6 +1085,8 @@ static void *sdbWorkerFp(void *pWorker) { int32_t qtype; void * unUsed; + taosBlockSIGPIPE(); + while (1) { int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed); if (numOfMsgs == 0) { diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 3c1c92226a26d0d0195020c05b579c0b6c326fa3..6b9f0e26a762092cfae02b5724d6cfedd24194d2 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -351,6 +351,8 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet, false); + dnodeGetClusterId(pConnectRsp->clusterId); + connect_over: if (code != TSDB_CODE_SUCCESS) { if (pConnectRsp) rpcFreeCont(pConnectRsp); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index a5d7729ec4a77273f99ad5deac70871fe011dc9f..26e60cdc9cfae17a53e14ad3041ffc058883d1fb 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -26,6 +26,7 @@ #include "tcompare.h" #include "tdataformat.h" #include "tgrant.h" +#include "tqueue.h" #include "hash.h" #include "mnode.h" #include "dnode.h" @@ -720,6 +721,133 @@ static void mnodeExtractTableName(char* tableId, char* name) { } } +static SMnodeMsg *mnodeCreateSubMsg(SMnodeMsg *pBatchMasterMsg, int32_t contSize) { + SMnodeMsg *pSubMsg = taosAllocateQitem(sizeof(*pBatchMasterMsg) + contSize); + *pSubMsg = *pBatchMasterMsg; + + //pSubMsg->pCont = (char *) pSubMsg + sizeof(SMnodeMsg); + pSubMsg->rpcMsg.pCont = pSubMsg->pCont; + pSubMsg->successed = 0; + pSubMsg->expected = 0; + SCMCreateTableMsg *pCM = pSubMsg->rpcMsg.pCont; + pCM->numOfTables = htonl(1); + pCM->contLen = htonl(contSize); + + return pSubMsg; +} + +void mnodeDestroySubMsg(SMnodeMsg *pSubMsg) { + if (pSubMsg) { + // pUser is retained in batch master msg + if (pSubMsg->pDb) mnodeDecDbRef(pSubMsg->pDb); + if (pSubMsg->pVgroup) mnodeDecVgroupRef(pSubMsg->pVgroup); + if (pSubMsg->pTable) mnodeDecTableRef(pSubMsg->pTable); + if (pSubMsg->pSTable) mnodeDecTableRef(pSubMsg->pSTable); + if (pSubMsg->pAcct) mnodeDecAcctRef(pSubMsg->pAcct); + if (pSubMsg->pDnode) mnodeDecDnodeRef(pSubMsg->pDnode); + + taosFreeQitem(pSubMsg); + } +} + +static int32_t mnodeValidateCreateTableMsg(SCreateTableMsg *pCreateTable, SMnodeMsg *pMsg) { + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreateTable->db); + if (pMsg->pDb == NULL) { + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + return TSDB_CODE_MND_DB_NOT_SELECTED; + } + + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { + mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); + return TSDB_CODE_MND_DB_IN_DROPPING; + } + + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreateTable->tableId); + if (pMsg->pTable != NULL && pMsg->retry == 0) { + if (pCreateTable->getMeta) { + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + return mnodeGetChildTableMeta(pMsg); + } else if (pCreateTable->igExists) { + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + return TSDB_CODE_SUCCESS; + } else { + mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableId); + return TSDB_CODE_MND_TABLE_ALREADY_EXIST; + } + } + + if (pCreateTable->numOfTags != 0) { + mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableId, pMsg->rpcMsg.handle); + return mnodeProcessCreateSuperTableMsg(pMsg); + } else { + mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableId, pMsg->rpcMsg.handle); + return mnodeProcessCreateChildTableMsg(pMsg); + } +} + +static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) { + if (pMsg->pBatchMasterMsg == NULL) { // batch master first round + pMsg->pBatchMasterMsg = pMsg; + + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + int32_t numOfTables = htonl(pCreate->numOfTables); + int32_t contentLen = htonl(pCreate->contLen); + pMsg->expected = numOfTables; + + int32_t code = TSDB_CODE_SUCCESS; + SCreateTableMsg *pCreateTable = (SCreateTableMsg*) ((char*) pCreate + sizeof(SCMCreateTableMsg)); + for (SCreateTableMsg *p = pCreateTable; p < (SCreateTableMsg *) ((char *) pCreate + contentLen); p = (SCreateTableMsg *) ((char *) p + htonl(p->len))) { + SMnodeMsg *pSubMsg = mnodeCreateSubMsg(pMsg, sizeof(SCMCreateTableMsg) + htonl(p->len)); + memcpy(pSubMsg->pCont + sizeof(SCMCreateTableMsg), p, htonl(p->len)); + code = mnodeValidateCreateTableMsg(p, pSubMsg); + + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + ++pSubMsg->pBatchMasterMsg->successed; + mnodeDestroySubMsg(pSubMsg); + continue; + } + + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mnodeDestroySubMsg(pSubMsg); + return code; + } + } + + if (pMsg->successed >= pMsg->expected) { + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + } else { + if (pMsg->pBatchMasterMsg != pMsg) { // batch sub replay + SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; + SCreateTableMsg *pCreateTable = (SCreateTableMsg*) ((char*) pCreate + sizeof(SCMCreateTableMsg)); + int32_t code = mnodeValidateCreateTableMsg(pCreateTable, pMsg); + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + ++pMsg->pBatchMasterMsg->successed; + mnodeDestroySubMsg(pMsg); + } + + if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mnodeDestroySubMsg(pMsg); + return code; + } + + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + return code; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + } else { // batch master replay, reprocess the whole batch + assert(0); + } + } +} + static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; @@ -729,6 +857,11 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { // todo return error } + // batch master msg first round or reprocessing and batch sub msg reprocessing + if (numOfTables > 1 || pMsg->pBatchMasterMsg != NULL) { + return mnodeProcessBatchCreateTableMsg(pMsg); + } + SCreateTableMsg *p = (SCreateTableMsg*)((char*) pCreate + sizeof(SCMCreateTableMsg)); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(p->db); if (pMsg->pDb == NULL) { @@ -1737,6 +1870,18 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { mDebug("msg:%p, app:%p table:%s, created in dnode, thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->rpcMsg.handle); + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->successed; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -2171,11 +2316,12 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); - if (pTable->superTable != NULL) { - tstrncpy(pMeta->sTableId, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); - } - if (pTable->info.type == TSDB_CHILD_TABLE && pTable->superTable != NULL) { + if (pTable->info.type == TSDB_CHILD_TABLE) { + assert(pTable->superTable != NULL); + tstrncpy(pMeta->sTableName, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN); + + pMeta->suid = pTable->superTable->uid; pMeta->sversion = htons(pTable->superTable->sversion); pMeta->tversion = htons(pTable->superTable->tversion); pMeta->numOfTags = (int8_t)pTable->superTable->numOfTags; @@ -2477,6 +2623,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeSendDropChildTableMsg(pMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->successed; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); return; } @@ -2494,6 +2653,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { pMsg->pTable = NULL; mnodeDestroyChildTable(pTable); + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, code); } } else { @@ -2519,6 +2691,19 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { //Avoid retry again in client rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY; } + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return; + } + dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code); } } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 8e9bf97496d80ac9b4d05264372bead0e0a1fb51..827be0687dbac338d660d07fefaf563d5c5d0341 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -184,6 +184,7 @@ static int32_t mnodeVgroupActionEncode(SSdbRow *pRow) { for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { pTmpVgroup->vnodeGid[i].pDnode = NULL; pTmpVgroup->vnodeGid[i].role = 0; + memset(pTmpVgroup->vnodeGid[i].vver, 0, sizeof(pTmpVgroup->vnodeGid[i].vver)); } pRow->rowSize = tsVgUpdateSize; @@ -317,9 +318,10 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; if (pVgid->pDnode == pDnode) { - mTrace("dnode:%d, receive status from dnode, vgId:%d status:%s last:%s", pDnode->dnodeId, pVgroup->vgId, - syncRole[pVload->role], syncRole[pVgid->role]); + mTrace("vgId:%d, receive vnode status from dnode:%d, status:%s last:%s vver:%" PRIu64, pVgroup->vgId, + pDnode->dnodeId, syncRole[pVload->role], syncRole[pVgid->role], pVload->vnodeVersion); pVgid->role = pVload->role; + mnodeSetVgidVer(pVgid->vver, pVload->vnodeVersion); if (pVload->role == TAOS_SYNC_ROLE_MASTER) { pVgroup->inUse = i; } @@ -659,7 +661,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "onlineVnodes"); + strcpy(pSchema[cols].name, "onlines"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -674,13 +676,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } @@ -859,10 +861,12 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->precision = pDb->cfg.precision; pCfg->compression = pDb->cfg.compression; pCfg->walLevel = pDb->cfg.walLevel; - pCfg->replications = (int8_t) pVgroup->numOfVnodes; + pCfg->vgReplica = (int8_t) pVgroup->numOfVnodes; pCfg->wals = 3; pCfg->quorum = pDb->cfg.quorum; pCfg->update = pDb->cfg.update; + pCfg->cacheLastRow = pDb->cfg.cacheLastRow; + pCfg->dbReplica = pDb->cfg.replications; SVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { @@ -982,6 +986,19 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); + + if (mnodeMsg->pBatchMasterMsg) { + ++mnodeMsg->pBatchMasterMsg->received; + if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received + >= mnodeMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(mnodeMsg); + + return; + } + dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { @@ -991,6 +1008,19 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { .pObj = pVgroup }; sdbDeleteRow(&row); + + if (mnodeMsg->pBatchMasterMsg) { + ++mnodeMsg->pBatchMasterMsg->received; + if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received + >= mnodeMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, mnodeMsg->code); + } + + mnodeDestroySubMsg(mnodeMsg); + + return; + } + dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); } } @@ -1178,3 +1208,14 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) { mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups); } + +int32_t mnodeGetVgidVer(int8_t *cver) { + int32_t iver = ((int32_t)cver[0]) * 10000 + ((int32_t)cver[1]) * 100 + (int32_t)cver[2]; + return iver; +} + +void mnodeSetVgidVer(int8_t *cver, uint64_t iver) { + cver[0] = (int8_t)((int32_t)(iver % 1000000) / 10000); + cver[1] = (int8_t)((int32_t)(iver % 100000) / 100); + cver[2] = (int8_t)(iver % 100); +} diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index a71e74e97f4d9910414a5e801b89a1968d1df050..74e1bd487815942651111a2aa85e31650281bf20 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -29,12 +29,13 @@ extern "C" { #endif // TAOS_OS_FUNC_SEMPHONE_PTHREAD -bool taosCheckPthreadValid(pthread_t thread); -int64_t taosGetPthreadId(); -void taosResetPthread(pthread_t *thread); -bool taosComparePthread(pthread_t first, pthread_t second); +bool taosCheckPthreadValid(pthread_t thread); +int64_t taosGetSelfPthreadId(); +int64_t taosGetPthreadId(pthread_t thread); +void taosResetPthread(pthread_t* thread); +bool taosComparePthread(pthread_t first, pthread_t second); int32_t taosGetPId(); -int32_t taosGetCurrentAPPName(char *name, int32_t* len); +int32_t taosGetCurrentAPPName(char* name, int32_t* len); #ifdef __cplusplus } diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index baf7687dd03e1ee4f6dd92e3204244b3d31b7a1f..13d3fa407921ebf93992d3d0da5c8dabc41f5e44 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -59,6 +59,7 @@ extern "C" { // TAOS_OS_FUNC_SOCKET int32_t taosSetNonblocking(SOCKET sock, int32_t on); +void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 9eb8c18a40a11fac9ada037163011cdcf92201fb..d379e56ed83bd19e4b9ff5336e95cd019f296a36 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -31,7 +31,8 @@ int tsem_wait(tsem_t* sem) { #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } -int64_t taosGetPthreadId() { return (int64_t)pthread_self(); } +int64_t taosGetSelfPthreadId() { return (int64_t)pthread_self(); } +int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } void taosResetPthread(pthread_t *thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } int32_t taosGetPId() { return getpid(); } diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index c7c9d774271555ae5989aa18b7e00d325a0eddde..729471247f884977e65d86166cabf06641581e2f 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -39,6 +39,10 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() { + signal(SIGPIPE, SIG_IGN); +} + void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemphone.c index 1f723540f695e4a4609a9ec74e773a1559e7f30a..0bc760b35e9016f63a8cd40e1035891cf36e27ae 100644 --- a/src/os/src/windows/wSemphone.c +++ b/src/os/src/windows/wSemphone.c @@ -25,14 +25,16 @@ bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } void taosResetPthread(pthread_t *thread) { thread->p = 0; } -int64_t taosGetPthreadId() { +int64_t taosGetPthreadId(pthread_t thread) { #ifdef PTW32_VERSION - return pthread_getw32threadid_np(pthread_self()); + return pthread_getw32threadid_np(thread); #else - return (int64_t)pthread_self(); + return (int64_t)thread; #endif } +int64_t taosGetSelfPthreadId() { return taosGetPthreadId(pthread_self()); } + bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; } diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 3b091b269931e644d9f8c2c01ba3c9cb9ddc520c..9697c5e65fa374255c82596ec99d0c8b3e5b9aad 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -46,6 +46,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { return 0; } +void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 755dad2d850c24192f5eee35c49492378ba8fb73..72604e79b710f09e67fde825915db63bc62243d9 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -50,9 +50,16 @@ static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char * char head[512] = {0}; char body[512] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[pContext->parser->httpVersion], - httpCode, httpCodeStr, httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[httpVersion], httpCode, + httpCodeStr, httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); @@ -136,7 +143,7 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { else httpCode = 400; - if (pContext->parser->httpCode != 0) { + if (pContext->parser && pContext->parser->httpCode != 0) { httpCode = pContext->parser->httpCode; } @@ -164,9 +171,16 @@ void httpSendSuccResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->parser->httpVersion], - httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[httpVersion], + httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); @@ -177,9 +191,16 @@ void httpSendOptionResp(HttpContext *pContext, char *desc) { char head[1024] = {0}; char body[1024] = {0}; + int8_t httpVersion = 0; + int8_t keepAlive = 0; + if (pContext->parser != NULL) { + httpVersion = pContext->parser->httpVersion; + keepAlive = pContext->parser->keepAlive; + } + int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc); - int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->parser->httpVersion], - httpKeepAliveStr[pContext->parser->keepAlive], bodyLen); + int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[httpVersion], + httpKeepAliveStr[keepAlive], bodyLen); httpWriteBuf(pContext, head, headLen); httpWriteBuf(pContext, body, bodyLen); diff --git a/src/query/inc/qSyntaxtreefunction.h b/src/query/inc/qArithmeticOperator.h similarity index 91% rename from src/query/inc/qSyntaxtreefunction.h rename to src/query/inc/qArithmeticOperator.h index 6f91d2f7eded38425cf8414b6998e0162557b2c8..f13c63acc3509b13b1696c6360d4135e00079ba2 100644 --- a/src/query/inc/qSyntaxtreefunction.h +++ b/src/query/inc/qArithmeticOperator.h @@ -23,7 +23,7 @@ extern "C" { typedef void (*_bi_consumer_fn_t)(void *left, void *right, int32_t numOfLeft, int32_t numOfRight, void *output, int32_t order); -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr); +_bi_consumer_fn_t getArithmeticOperatorFn(int32_t leftType, int32_t rightType, int32_t optr); #ifdef __cplusplus } diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 28c1c7b838236d10a3f67fa7bbfc15a9b36c4612..39af7261efc222c8a6bcfc809288c256eccb0970 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -74,9 +74,7 @@ typedef struct tExprNode { }; } tExprNode; -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); - -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); tExprNode* exprTreeFromBinary(const void* data, size_t size); @@ -87,6 +85,8 @@ void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index d1278e2eee173dd819c0aae582d5ddef2a5543af..32a68549fabd73104c56ea89a705f7cf5a6a30d1 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,13 +33,6 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -typedef struct SGroupResInfo { - int32_t groupId; - int32_t numOfDataPages; - int32_t pageId; - int32_t rowId; -} SGroupResInfo; - typedef struct SResultRowPool { int32_t elemSize; int32_t blockSize; @@ -72,6 +65,12 @@ typedef struct SResultRow { union {STimeWindow win; char* key;}; // start key of current time window } SResultRow; +typedef struct SGroupResInfo { + int32_t rowId; + int32_t index; + SArray* pRows; // SArray +} SGroupResInfo; + /** * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. @@ -89,7 +88,6 @@ typedef struct SResultRowInfo { int32_t size:24; // number of result set int32_t capacity; // max capacity int32_t curIndex; // current start active index - int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key } SResultRowInfo; @@ -192,7 +190,7 @@ typedef struct SQueryRuntimeEnv { void* pSecQueryHandle; // another thread for bool stableQuery; // super table query or not bool topBotQuery; // TODO used bitwise flag - bool groupbyNormalCol; // denote if this is a groupby normal column query + bool groupbyColumn; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not bool timeWindowInterpo;// if the time window start/end required interpolation bool queryWindowIdentical; // all query time windows are identical for all tables in one group @@ -206,6 +204,8 @@ typedef struct SQueryRuntimeEnv { int32_t* rowCellInfoOffset;// offset value for each row result cell info char** prevRow; char** nextRow; + + SArithmeticSupport *sasArray; } SQueryRuntimeEnv; enum { @@ -239,6 +239,7 @@ typedef struct SQInfo { int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context int64_t startExecTs; // start to exec timestamp + char* sql; // query sql string } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/qHistogram.h b/src/query/inc/qHistogram.h index 442e61750b06d269ef48c5e74ede7646f7ac62e1..7742d151a06456a8b3c10e8b1f0aeba6203a28b7 100644 --- a/src/query/inc/qHistogram.h +++ b/src/query/inc/qHistogram.h @@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto); void tHistogramPrint(SHistogramInfo* pHisto); -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val); +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val); SHeapEntry* tHeapCreate(int32_t numOfEntries); void tHeapSort(SHeapEntry* pEntry, int32_t len); diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index ee72500241a8ad8d5002e3d921d46d5d0187c453..56e676ef167b8c9f1a694ef0571d1b0318ef6992 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -120,7 +120,8 @@ typedef struct SCreateDBInfo { int32_t compressionLevel; SStrToken precision; bool ignoreExists; - int8_t update; + int8_t update; + int8_t cachelast; SArray *keep; } SCreateDBInfo; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index dde2e398459ebef7f870cd5f4482f5755247ecf9..4620e3d61e5e57c1072a7c3b3f6ca5face080d37 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -34,17 +34,13 @@ int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); void resetResultRowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); -void popFrontResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int32_t num); -void clearClosedResultRows(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pResultRowInfo); int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); -void removeRedundantResultRows(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order); int32_t initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); -void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); @@ -77,7 +73,6 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); bool isPointInterpoQuery(SQuery *pQuery); diff --git a/src/query/inc/queryLog.h b/src/query/inc/queryLog.h index 825ff12538eab30a98765c2d61a70c865a1cb979..d4e909d33a5f79ee11de5e13b21ada2930a738bf 100644 --- a/src/query/inc/queryLog.h +++ b/src/query/inc/queryLog.h @@ -22,8 +22,8 @@ extern "C" { #include "tlog.h" -extern uint32_t qDebugFlag; -extern uint32_t tscEmbedded; +extern int32_t qDebugFlag; +extern int8_t tscEmbedded; #define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", 255, __VA_ARGS__); }} while(0) #define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", 255, __VA_ARGS__); }} while(0) diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 0de6027006f94fa6b263bab91a9d2d0e752d5f7d..dda15fb508d17bfb80ba95d2d9249fc0021d4ae8 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -112,29 +112,29 @@ cmd ::= SHOW dbPrefix(X) STABLES. { cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { SStrToken token; - setDBName(&token, &X); + setDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } //drop configure for tables cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { X.n += Z.n; - setDropDBTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y); + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y); } -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDBTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y); } +cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y); } cmd ::= DROP DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } cmd ::= DROP USER ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } cmd ::= DROP ACCOUNT ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } @@ -149,16 +149,16 @@ cmd ::= DESCRIBE ids(X) cpxName(Y). { } /////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// -cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } -cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSQL(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} +cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } +cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} -cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} -cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} +cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} +cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. @@ -179,9 +179,9 @@ ifnotexists(X) ::= . { X.n = 0;} //create option for dnode/db/user/account cmd ::= CREATE DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). - { setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} + { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} -cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSQL(pInfo, &X, &Y);} +cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} pps(Y) ::= . { Y.n = 0; } pps(Y) ::= PPS INTEGER(X). { Y = X; } @@ -240,6 +240,7 @@ fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; } update(Y) ::= UPDATE INTEGER(X). { Y = X; } +cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } %type db_optr {SCreateDBInfo} db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} @@ -258,6 +259,7 @@ db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strto db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } %type alter_db_optr {SCreateDBInfo} alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} @@ -270,21 +272,22 @@ alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLeve alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { X.type = 0; - tSQLSetColumnType (&A, &X); + tSqlSetColumnType (&A, &X); } //define binary type, e.g., binary(10), nchar(10) typename(A) ::= ids(X) LP signed(Y) RP. { if (Y <= 0) { X.type = 0; - tSQLSetColumnType(&A, &X); + tSqlSetColumnType(&A, &X); } else { X.type = -Y; // negative value of name length - tSQLSetColumnType(&A, &X); + tSqlSetColumnType(&A, &X); } } @@ -315,8 +318,8 @@ create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { %type create_table_args{SCreateTableSQL*} create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { - A = tSetCreateSQLElems(X, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + A = tSetCreateSqlElems(X, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); V.n += Z.n; setCreatedTableName(pInfo, &V, &U); @@ -324,8 +327,8 @@ create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { // create super table create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateSQLElems(X, Y, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + A = tSetCreateSqlElems(X, Y, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); V.n += Z.n; setCreatedTableName(pInfo, &V, &U); @@ -343,8 +346,8 @@ create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName( // create stream // create table table_name as select count(*) from super_table_name interval(time) create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { - A = tSetCreateSQLElems(NULL, NULL, S, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); + A = tSetCreateSqlElems(NULL, NULL, S, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); V.n += Z.n; setCreatedTableName(pInfo, &V, &U); @@ -359,7 +362,7 @@ columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(T // The information used for a column is the name and type of column: // tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) column(A) ::= ids(X) typename(Y). { - tSQLSetColumnInfo(&A, &X, &Y); + tSqlSetColumnInfo(&A, &X, &Y); } %type tagitemlist {SArray*} @@ -407,7 +410,7 @@ tagitem(A) ::= PLUS(X) FLOAT(Y). { %type select {SQuerySQL*} %destructor select {doDestroyQuerySql($$);} select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { - A = tSetQuerySQLElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); + A = tSetQuerySqlElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); } %type union {SSubclauseInfo*} @@ -418,33 +421,33 @@ union(Y) ::= LP union(X) RP. { Y = X; } union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); } union(Y) ::= union(Z) UNION ALL LP select(X) RP. { Y = appendSelectClause(Z, X); } -cmd ::= union(X). { setSQLInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } +cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } // Support for the SQL exprssion without from & where subclauses, e.g., // select current_database(), // select server_version(), select client_version(), // select server_state(); select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySQLElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + A = tSetQuerySqlElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } // selcollist is a list of expressions that are to become the return // values of the SELECT statement. The "*" in statements like // "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. %type selcollist {tSQLExprList*} -%destructor selcollist {tSQLExprListDestroy($$);} +%destructor selcollist {tSqlExprListDestroy($$);} %type sclp {tSQLExprList*} -%destructor sclp {tSQLExprListDestroy($$);} +%destructor sclp {tSqlExprListDestroy($$);} sclp(A) ::= selcollist(X) COMMA. {A = X;} sclp(A) ::= . {A = 0;} selcollist(A) ::= sclp(P) expr(X) as(Y). { - A = tSQLExprListAppend(P, X, Y.n?&Y:0); + A = tSqlExprListAppend(P, X, Y.n?&Y:0); } selcollist(A) ::= sclp(P) STAR. { - tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - A = tSQLExprListAppend(P, pNode, 0); + tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); + A = tSqlExprListAppend(P, pNode, 0); } // An option "AS " phrase that can follow one of the expressions that @@ -573,7 +576,7 @@ grouplist(A) ::= item(X). { //having clause, ignore the input condition in having %type having_opt {tSQLExpr*} -%destructor having_opt {tSQLExprDestroy($$);} +%destructor having_opt {tSqlExprDestroy($$);} having_opt(A) ::=. {A = 0;} having_opt(A) ::= HAVING expr(X). {A = X;} @@ -595,7 +598,7 @@ slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). {A.limit = Y; A.offset = X;} %type where_opt {tSQLExpr*} -%destructor where_opt {tSQLExprDestroy($$);} +%destructor where_opt {tSqlExprDestroy($$);} where_opt(A) ::= . {A = 0;} where_opt(A) ::= WHERE expr(X). {A = X;} @@ -603,67 +606,67 @@ where_opt(A) ::= WHERE expr(X). {A = X;} /////////////////////////// Expression Processing ///////////////////////////// // %type expr {tSQLExpr*} -%destructor expr {tSQLExprDestroy($$);} +%destructor expr {tSqlExprDestroy($$);} expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->token.z = X.z; A->token.n = (Z.z - X.z + 1);} -expr(A) ::= ID(X). { A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). { A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). { A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= STRING(X). { A = tSQLExprIdValueCreate(&X, TK_STRING);} -expr(A) ::= NOW(X). { A = tSQLExprIdValueCreate(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). { A = tSQLExprIdValueCreate(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). { A = tSQLExprIdValueCreate(&X, TK_BOOL);} +expr(A) ::= ID(X). { A = tSqlExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ID);} +expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ALL);} + +expr(A) ::= INTEGER(X). { A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} +expr(A) ::= FLOAT(X). { A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} +expr(A) ::= STRING(X). { A = tSqlExprIdValueCreate(&X, TK_STRING);} +expr(A) ::= NOW(X). { A = tSqlExprIdValueCreate(&X, TK_NOW); } +expr(A) ::= VARIABLE(X). { A = tSqlExprIdValueCreate(&X, TK_VARIABLE);} +expr(A) ::= BOOL(X). { A = tSqlExprIdValueCreate(&X, TK_BOOL);} // ordinary functions: min(x), max(x), top(k, 20) -expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSQLExprCreateFunction(Y, &X, &E, X.type); } +expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); } // for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation -expr(A) ::= ID(X) LP STAR RP(Y). { A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); } +expr(A) ::= ID(X) LP STAR RP(Y). { A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); } // is (not) null expression -expr(A) ::= expr(X) IS NULL. {A = tSQLExprCreate(X, NULL, TK_ISNULL);} -expr(A) ::= expr(X) IS NOT NULL. {A = tSQLExprCreate(X, NULL, TK_NOTNULL);} +expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);} +expr(A) ::= expr(X) IS NOT NULL. {A = tSqlExprCreate(X, NULL, TK_NOTNULL);} // relational expression -expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);} -expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);} -expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);} -expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);} -expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);} -expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);} +expr(A) ::= expr(X) LT expr(Y). {A = tSqlExprCreate(X, Y, TK_LT);} +expr(A) ::= expr(X) GT expr(Y). {A = tSqlExprCreate(X, Y, TK_GT);} +expr(A) ::= expr(X) LE expr(Y). {A = tSqlExprCreate(X, Y, TK_LE);} +expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} +expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} +expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} -expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);} -expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); } +expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} +expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } // binary arithmetic expression -expr(A) ::= expr(X) PLUS expr(Y). {A = tSQLExprCreate(X, Y, TK_PLUS); } -expr(A) ::= expr(X) MINUS expr(Y). {A = tSQLExprCreate(X, Y, TK_MINUS); } -expr(A) ::= expr(X) STAR expr(Y). {A = tSQLExprCreate(X, Y, TK_STAR); } -expr(A) ::= expr(X) SLASH expr(Y). {A = tSQLExprCreate(X, Y, TK_DIVIDE);} -expr(A) ::= expr(X) REM expr(Y). {A = tSQLExprCreate(X, Y, TK_REM); } +expr(A) ::= expr(X) PLUS expr(Y). {A = tSqlExprCreate(X, Y, TK_PLUS); } +expr(A) ::= expr(X) MINUS expr(Y). {A = tSqlExprCreate(X, Y, TK_MINUS); } +expr(A) ::= expr(X) STAR expr(Y). {A = tSqlExprCreate(X, Y, TK_STAR); } +expr(A) ::= expr(X) SLASH expr(Y). {A = tSqlExprCreate(X, Y, TK_DIVIDE);} +expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } // like expression -expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); } +expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } //in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, TK_IN); } +expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSQLExpr*)Y, TK_IN); } %type exprlist {tSQLExprList*} -%destructor exprlist {tSQLExprListDestroy($$);} +%destructor exprlist {tSqlExprListDestroy($$);} %type expritem {tSQLExpr*} -%destructor expritem {tSQLExprDestroy($$);} +%destructor expritem {tSqlExprDestroy($$);} -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSQLExprListAppend(X,Y,0);} -exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);} +exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0);} +exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0);} expritem(A) ::= expr(X). {A = X;} expritem(A) ::= . {A = 0;} @@ -673,8 +676,8 @@ cmd ::= RESET QUERY CACHE. { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} ///////////////////////////////////ALTER TABLE statement////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { X.n += F.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { @@ -683,15 +686,15 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { toTSDBType(A.type); SArray* K = tVariantListAppendToken(NULL, &A, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } //////////////////////////////////ALTER TAGS statement///////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { X.n += Y.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { X.n += Z.n; @@ -699,8 +702,8 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { toTSDBType(Y.type); SArray* A = tVariantListAppendToken(NULL, &Y, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { @@ -712,8 +715,8 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { toTSDBType(Z.type); A = tVariantListAppendToken(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { @@ -723,14 +726,14 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { SArray* A = tVariantListAppendToken(NULL, &Y, -1); A = tVariantListAppend(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); - setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } ////////////////////////////////////////kill statement/////////////////////////////////////// -cmd ::= KILL CONNECTION INTEGER(Y). {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} -cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &X);} -cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &X);} +cmd ::= KILL CONNECTION INTEGER(Y). {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &Y);} +cmd ::= KILL STREAM INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &X);} +cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &X);} %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 38bb0b8a71cbc4d219fd39c209a812660dd0a1b7..51048bbe72368cb6869092b65d5f035f26f3545b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -112,11 +112,10 @@ extern "C" { #define TOP_BOTTOM_QUERY_LIMIT 100 enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - FIRST_STAGE_MERGE = 0x10u, - SECONDARY_STAGE_MERGE = 0x20u, + MASTER_SCAN = 0x0u, + REVERSE_SCAN = 0x1u, + REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan + MERGE_STAGE = 0x20u, }; #define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) @@ -191,8 +190,8 @@ typedef struct SQLFunctionCtx { int64_t nStartQueryTimestamp; // timestamp range of current query when function is executed on a specific data block int32_t numOfParams; tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */ - int64_t * ptsList; // corresponding timestamp array list - void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + int64_t *ptsList; // corresponding timestamp array list + void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; tVariant tag; @@ -215,18 +214,12 @@ typedef struct SQLAggFuncElem { void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version - // some sql function require scan data twice or more, e.g.,stddev + // some sql function require scan data twice or more, e.g.,stddev, percentile void (*xNextStep)(SQLFunctionCtx *pCtx); - /* - * finalizer must be called after all xFunction has been executed to - * generated final result. Otherwise, the value in aOutputBuf is a intern result. - */ + // finalizer must be called after all xFunction has been executed to generated final result. void (*xFinalize)(SQLFunctionCtx *pCtx); - - void (*distMergeFunc)(SQLFunctionCtx *pCtx); - - void (*distSecondaryMergeFunc)(SQLFunctionCtx *pCtx); + void (*mergeFunc)(SQLFunctionCtx *pCtx); int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); } SQLAggFuncElem; diff --git a/src/client/src/tscFunctionImpl.c b/src/query/src/qAggMain.c similarity index 87% rename from src/client/src/tscFunctionImpl.c rename to src/query/src/qAggMain.c index f313d355f118a4357960868b912872c4124b218b..2997d56326b664488a1942df0150343bb5b0f6f9 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/query/src/qAggMain.c @@ -15,27 +15,26 @@ #include "os.h" #include "qAst.h" -#include "qExtbuffer.h" #include "qFill.h" #include "qHistogram.h" #include "qPercentile.h" -#include "qSyntaxtreefunction.h" #include "qTsbuf.h" #include "taosdef.h" #include "taosmsg.h" #include "tscLog.h" #include "tscSubquery.h" -#include "tscompression.h" #include "tsqlfunction.h" -#include "tutil.h" #include "ttype.h" -#define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) -#define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes) +#define GET_INPUT_DATA_LIST(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) +#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) + +#define GET_TS_LIST(x) ((TSKEY*)&((x)->ptsList[(x)->startOffset])) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) #define GET_TRUE_DATA_TYPE() \ int32_t type = 0; \ - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { \ + if (pCtx->currentStage == MERGE_STAGE) { \ type = pCtx->outputType; \ assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ } else { \ @@ -62,7 +61,7 @@ } \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ } \ - } while (0); + } while (0) #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ @@ -395,7 +394,7 @@ static void count_function(SQLFunctionCtx *pCtx) { } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { - char *val = GET_INPUT_CHAR_INDEX(pCtx, i); + char *val = GET_INPUT_DATA(pCtx, i); if (isNull(val, pCtx->inputType)) { continue; } @@ -420,7 +419,7 @@ static void count_function(SQLFunctionCtx *pCtx) { } static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -434,7 +433,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void count_func_merge(SQLFunctionCtx *pCtx) { - int64_t *pData = (int64_t *)GET_INPUT_CHAR(pCtx); + int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->aOutputBuf) += pData[i]; } @@ -498,7 +497,7 @@ int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ continue; \ } \ - TSKEY key = (ctx)->ptsList[i]; \ + TSKEY key = GET_TS_DATA(ctx, i); \ UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ } @@ -525,7 +524,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { *retVal += GET_DOUBLE_VAL((const char*)&(pCtx->preAggVals.statis.sum)); } } else { // computing based on the true data block - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { @@ -558,7 +557,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { } static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -608,21 +607,21 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { +static void sum_func_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + GET_TRUE_DATA_TYPE(); assert(pCtx->stableQuery); - + for (int32_t i = 0; i < pCtx->size; ++i) { - char * input = GET_INPUT_CHAR_INDEX(pCtx, i); + char * input = GET_INPUT_DATA(pCtx, i); SSumInfo *pInput = (SSumInfo *)input; if (pInput->hasResult != DATA_SET_FLAG) { continue; } - + notNullElems++; - + switch (type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -637,25 +636,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { } } } - - return notNullElems; -} -static void sum_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - - SET_VAL(pCtx, notNullElems, 1); - SSumInfo *pSumInfo = (SSumInfo *)pCtx->aOutputBuf; - - if (notNullElems > 0) { - // pCtx->numOfIteratedElems += notNullElems; - pSumInfo->hasResult = DATA_SET_FLAG; - } -} - -static void sum_func_second_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = sum_merge_impl(pCtx); - SET_VAL(pCtx, notNullElems, 1); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -764,7 +745,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->preAggVals.statis.sum)); } } else { - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); @@ -799,7 +780,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -839,33 +820,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current buffer is null - continue; - } - - pAvgInfo->sum += pInput->sum; - pAvgInfo->num += pInput->num; - } - - // if the data set hasResult is not set, the result is null - if (pAvgInfo->num > 0) { - pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_second_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->aOutputBuf; - char * input = GET_INPUT_CHAR(pCtx); + char * input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SAvgInfo *pInput = (SAvgInfo *)input; @@ -886,7 +843,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { static void avg_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { @@ -920,7 +877,15 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (pCtx->preAggVals.isSet) { *notNullElems = pCtx->size - pCtx->preAggVals.statis.numOfNull; assert(*notNullElems >= 0); + + if (*notNullElems == 0) { + return; + } + if (*notNullElems == 0){ + return; + } + void * tval = NULL; int16_t index = 0; @@ -943,7 +908,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if (index < 0 || index >= pCtx->size + pCtx->startOffset) { index = 0; } - + + // the index is the original position, not the relative position key = pCtx->ptsList[index]; } @@ -994,7 +960,9 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, return; } - void *p = GET_INPUT_CHAR(pCtx); + void *p = GET_INPUT_DATA_LIST(pCtx); + TSKEY *tsList = GET_TS_LIST(pCtx); + *notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { @@ -1013,7 +981,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; - TSKEY k = pCtx->ptsList[i]; + TSKEY k = tsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); } @@ -1144,7 +1112,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_CHAR_INDEX(pCtx, i); + char *input = GET_INPUT_DATA(pCtx, i); if (input[bytes] != DATA_SET_FLAG) { continue; } @@ -1198,17 +1166,6 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp } static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 1); - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void min_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1220,16 +1177,6 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { } static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElems = minmax_merge_impl(pCtx, pCtx->inputBytes, pCtx->aOutputBuf, 0); - - SET_VAL(pCtx, numOfElems, 1); - if (numOfElems > 0) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; - *flag = DATA_SET_FLAG; - } -} - -static void max_func_second_merge(SQLFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1241,8 +1188,8 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { } static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); - TSKEY key = pCtx->ptsList[index]; + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY key = GET_TS_DATA(pCtx, index); int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -1281,7 +1228,7 @@ static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin } static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1297,7 +1244,7 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1330,7 +1277,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { double *retVal = &pStd->res; double avg = pStd->avg; - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { @@ -1381,7 +1328,7 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { avg_function_f(pCtx, index); } else { double avg = pStd->avg; - void * pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void * pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; @@ -1487,14 +1434,14 @@ static void first_function(SQLFunctionCtx *pCtx) { // handle the null value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - TSKEY k = pCtx->ptsList[i]; + TSKEY k = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, k); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1513,7 +1460,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1521,7 +1468,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1530,7 +1477,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); @@ -1561,7 +1508,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { // find the first not null value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -1579,7 +1526,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { } static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1594,26 +1541,9 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void first_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pInput->ts < pOutput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - -static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); - char * pData = GET_INPUT_CHAR(pCtx); + char * pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; @@ -1648,7 +1578,7 @@ static void last_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; @@ -1656,7 +1586,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[i]; + TSKEY ts = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -1671,7 +1601,7 @@ static void last_function(SQLFunctionCtx *pCtx) { } static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1685,7 +1615,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -1693,7 +1623,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->complete = true; // set query completed } else { // in case of ascending order check, all data needs to be checked SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); - TSKEY ts = pCtx->ptsList[index]; + TSKEY ts = GET_TS_DATA(pCtx, index); char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { @@ -1707,7 +1637,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = pCtx->ptsList; + int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); @@ -1741,7 +1671,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { if (!pCtx->requireNull) { continue; @@ -1765,7 +1695,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -1783,32 +1713,13 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); } -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); - - assert(pCtx->size == 1 && pCtx->stableQuery); - - // the input data is null - SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - SFirstLastInfo *pOutput = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - if (pOutput->hasResult != DATA_SET_FLAG || pOutput->ts < pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes + sizeof(SFirstLastInfo)); - - DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } -} - /* * in the secondary merge(local reduce), the output is limited by the * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { - char *pData = GET_INPUT_CHAR(pCtx); +static void last_dist_func_merge(SQLFunctionCtx *pCtx) { + char *pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1837,7 +1748,7 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { */ static void last_row_function(SQLFunctionCtx *pCtx) { assert(pCtx->size >= 1); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); // assign the last element in current data block assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); @@ -1848,12 +1759,13 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // set the result to final result buffer in case of super table query if (pCtx->stableQuery) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; + pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); } else { - DO_UPDATE_TAG_COLUMNS(pCtx, pCtx->ptsList[pCtx->size - 1]); + TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); + DO_UPDATE_TAG_COLUMNS(pCtx, ts); } SET_VAL(pCtx, pCtx->size, 1); @@ -1884,7 +1796,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 dst->timestamp = tsKey; int32_t size = 0; - if (stage == SECONDARY_STAGE_MERGE || stage == FIRST_STAGE_MERGE) { + if (stage == MERGE_STAGE) { memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { @@ -2147,7 +2059,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { return (STopBotInfo*) pCtx->aOutputBuf; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer return GET_ROWCELL_INTERBUF(pResInfo); @@ -2245,13 +2157,15 @@ static void top_function(SQLFunctionCtx *pCtx) { assert(pRes->num >= 0); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } notNullElems++; - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, 0); + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2268,7 +2182,7 @@ static void top_function(SQLFunctionCtx *pCtx) { } static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2277,36 +2191,16 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { assert(pRes->num >= 0); SET_VAL(pCtx, 1, 1); - do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + TSKEY ts = GET_TS_DATA(pCtx, index); + + do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void top_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void top_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2334,14 +2228,15 @@ static void bottom_function(SQLFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); + TSKEY ts = GET_TS_DATA(pCtx, i); + if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } notNullElems++; - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, pCtx->ptsList[i], pCtx->inputType, &pCtx->tagInfo, NULL, - 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2358,43 +2253,23 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); + TSKEY ts = GET_TS_DATA(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } STopBotInfo *pRes = getTopBotOutputInfo(pCtx); SET_VAL(pCtx, 1, 1); - do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, - NULL, 0); + do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } static void bottom_func_merge(SQLFunctionCtx *pCtx) { - char *input = GET_INPUT_CHAR(pCtx); - - STopBotInfo *pInput = (STopBotInfo *)input; - if (pInput->num <= 0) { - return; - } - - // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary - buildTopBotStruct(pInput, pCtx); - - assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - for (int32_t i = 0; i < pInput->num; ++i) { - do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->inputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); - } -} - -static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_CHAR(pCtx); + STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data buildTopBotStruct(pInput, pCtx); @@ -2490,7 +2365,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull); } else { for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2515,7 +2390,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { // the second stage, calculate the true percentile value for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2529,7 +2404,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { } static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2597,14 +2472,23 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////// +static void buildHistogramInfo(SAPercentileInfo* pInfo) { + pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); + pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); +} + static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return (SAPercentileInfo*) pCtx->aOutputBuf; + SAPercentileInfo* pInfo = NULL; + + if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { + pInfo = (SAPercentileInfo*) pCtx->aOutputBuf; } else { - return GET_ROWCELL_INTERBUF(pResInfo); + pInfo = GET_ROWCELL_INTERBUF(pResInfo); } + + buildHistogramInfo(pInfo); + return pInfo; } static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { @@ -2624,9 +2508,11 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); + + assert(pInfo->pHisto->elems != NULL); for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, i); + char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { continue; } @@ -2650,7 +2536,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2668,42 +2554,7 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - size_t size = sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); //(SAPercentileInfo *)pCtx->aOutputBuf; - SHistogramInfo * pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, size); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - - tHistogramDestroy(&pRes); - } - - SET_VAL(pCtx, 1, 1); - pResInfo->hasResult = DATA_SET_FLAG; -} - -static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); + SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); @@ -2738,7 +2589,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null assert(pOutput->pHisto->numOfElems > 0); @@ -2806,7 +2657,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { double(*param)[3] = pInfo->mat; double x = pInfo->startVal; - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); int32_t numOfElem = 0; switch (pCtx->inputType) { @@ -2866,7 +2717,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { } static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -2968,7 +2819,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { @@ -2993,7 +2844,7 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { } INC_INIT_VAL(pCtx, 1); - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); pCtx->aOutputBuf += pCtx->inputBytes; @@ -3047,7 +2898,7 @@ static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } @@ -3067,7 +2918,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) { // TODO difference in date column static void diff_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_CHAR(pCtx); + void *data = GET_INPUT_DATA_LIST(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); int32_t notNullElems = 0; @@ -3075,8 +2926,9 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - TSKEY * pTimestamp = pCtx->ptsOutputBuf; - + TSKEY* pTimestamp = pCtx->ptsOutputBuf; + TSKEY* tsList = GET_TS_LIST(pCtx); + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *pData = (int32_t *)data; @@ -3092,13 +2944,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64Key); // direct previous may be null - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3124,13 +2976,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pCtx->param[1].i64Key; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3156,12 +3008,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = pData[i] - pCtx->param[1].dKey; - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } @@ -3186,13 +3038,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (float)(pData[i] - pCtx->param[1].dKey); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3219,12 +3071,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3250,13 +3102,13 @@ static void diff_function(SQLFunctionCtx *pCtx) { pCtx->param[1].nType = pCtx->inputType; } else if ((i == 0 && pCtx->order == TSDB_ORDER_ASC) || (i == pCtx->size - 1 && pCtx->order == TSDB_ORDER_DESC)) { *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; } else { *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64Key); - *pTimestamp = pCtx->ptsList[i]; + *pTimestamp = tsList[i]; pOutput += 1; pTimestamp += 1; @@ -3297,12 +3149,12 @@ static void diff_function(SQLFunctionCtx *pCtx) { } else { \ *(type *)(ctx)->aOutputBuf = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64Key)); \ *(type *)(&(ctx)->param[1].i64Key) = *(type *)(d); \ - *(int64_t *)(ctx)->ptsOutputBuf = (ctx)->ptsList[index]; \ + *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ } \ } while (0); static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + char *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3322,7 +3174,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { } else { *(int32_t *)pCtx->aOutputBuf = *(int32_t *)pData - (int32_t)pCtx->param[1].i64Key; pCtx->param[1].i64Key = *(int32_t *)pData; - *(int64_t *)pCtx->ptsOutputBuf = pCtx->ptsList[index]; + *(int64_t *)pCtx->ptsOutputBuf = GET_TS_DATA(pCtx, index); } break; }; @@ -3375,7 +3227,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; pCtx->param[1].pz = NULL; @@ -3386,7 +3238,7 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - tExprTreeCalcTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); pCtx->aOutputBuf += pCtx->outputBytes; } @@ -3417,7 +3269,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { pCtx->param[0].dKey = DBL_MAX; pCtx->param[3].dKey = -DBL_MAX; } else { @@ -3466,7 +3318,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { goto _spread_over; } - void *pData = GET_INPUT_CHAR(pCtx); + void *pData = GET_INPUT_DATA_LIST(pCtx); numOfElems = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -3502,7 +3354,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { } static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -3544,45 +3396,12 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -void spread_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t notNullElems = 0; - for (int32_t i = 0; i < pCtx->size; ++i) { - SSpreadInfo *input = (SSpreadInfo *)GET_INPUT_CHAR_INDEX(pCtx, i); - - /* no assign tag, the value is null */ - if (input->hasResult != DATA_SET_FLAG) { - continue; - } - - if (pResData->min > input->min) { - pResData->min = input->min; - } - - if (pResData->max < input->max) { - pResData->max = input->max; - } - - pResData->hasResult = DATA_SET_FLAG; - notNullElems++; - } - - if (notNullElems > 0) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - pResInfo->hasResult = DATA_SET_FLAG; - } -} - /* * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer */ void spread_func_sec_merge(SQLFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_CHAR(pCtx); + SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); if (pData->hasResult != DATA_SET_FLAG) { return; } @@ -3605,7 +3424,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { */ SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { + if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (pResInfo->hasResult != DATA_SET_FLAG) { @@ -3660,26 +3479,25 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *tsList = GET_TS_LIST(pCtx); int32_t i = index; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); SPoint1* last = &pInfo->p; if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); + assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); assert(last->key == INT64_MIN); - last->key = primaryKey[tsIndex + i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); pInfo->dOutput += twa_get_area(pCtx->start, *last); @@ -3688,8 +3506,8 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t notNullElems++; i += step; } else if (pInfo->p.key == INT64_MIN) { - last->key = primaryKey[tsIndex + i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = last->key; @@ -3700,78 +3518,78 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t // calculate the value of switch(pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = (double) val[i]}; + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0); + float *val = (float*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0); + double *val = (double*) GET_INPUT_DATA(pCtx, 0); for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - SPoint1 st = {.key = primaryKey[i + tsIndex], .val = val[i]}; + SPoint1 st = {.key = tsList[i], .val = val[i]}; pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -3791,7 +3609,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t } static void twa_function(SQLFunctionCtx *pCtx) { - void *data = GET_INPUT_CHAR(pCtx); + void *data = GET_INPUT_DATA_LIST(pCtx); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3799,11 +3617,15 @@ static void twa_function(SQLFunctionCtx *pCtx) { // skip null value int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { + while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { i += step; } - int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size); + int32_t notNullElems = 0; + if (i >= 0 && i < pCtx->size) { + notNullElems = twa_function_impl(pCtx, i, pCtx->size); + } + SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3815,135 +3637,15 @@ static void twa_function(SQLFunctionCtx *pCtx) { } } -//TODO refactor static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - int32_t notNullElems = 0; - TSKEY *primaryKey = pCtx->ptsList; - + int32_t notNullElems = twa_function_impl(pCtx, index, 1); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - int32_t i = pCtx->startOffset; - int32_t size = pCtx->size; - - if (pCtx->start.key != INT64_MIN) { - assert(pInfo->p.key == INT64_MIN); - - pInfo->p.key = primaryKey[index]; - GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, pInfo->p); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += 1; - } else if (pInfo->p.key == INT64_MIN) { - pInfo->p.key = primaryKey[index]; - GET_TYPED_DATA(pInfo->p.val, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->p.key; - notNullElems++; - i += 1; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = (double) val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - - SPoint1 st = {.key = primaryKey[i + index], .val = val[i]}; - pInfo->dOutput += twa_get_area(pInfo->p, st);//((val[i] + pInfo->p.val) / 2) * (primaryKey[i + index] - pInfo->p.key); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end);//((pInfo->p.val + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->p.key); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3955,34 +3657,6 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -static void twa_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; - char * indicator = pCtx->aInputElemBuf; - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { - STwaInfo *pInput = (STwaInfo*) indicator; - - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } - - numOfNotNull++; - pBuf->dOutput += pInput->dOutput; - - pBuf->win = pInput->win; - pBuf->p = pInput->p; - } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; - } -} - /* * To copy the input to interResBuf to avoid the input buffer space be over writen * by next input data. The TWA function only applies to each table, so no merge procedure @@ -4025,8 +3699,10 @@ static void interp_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + assert(pCtx->startOffset == 0); + if (pCtx->size == 1) { - char *pData = GET_INPUT_CHAR(pCtx); + char *pData = GET_INPUT_DATA_LIST(pCtx); assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); } else { /* @@ -4052,13 +3728,13 @@ static void interp_function(SQLFunctionCtx *pCtx) { } else if (pInfo->type == TSDB_FILL_SET_VALUE) { tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType, true); } else if (pInfo->type == TSDB_FILL_PREV) { - char *data = GET_INPUT_CHAR_INDEX(pCtx, 0); + char *data = GET_INPUT_DATA(pCtx, 0); assignVal(pCtx->aOutputBuf, data, pCtx->outputBytes, pCtx->outputType); SET_VAL(pCtx, pCtx->size, 1); } else if (pInfo->type == TSDB_FILL_LINEAR) { - char *data1 = GET_INPUT_CHAR_INDEX(pCtx, 0); - char *data2 = GET_INPUT_CHAR_INDEX(pCtx, 1); + char *data1 = GET_INPUT_DATA(pCtx, 0); + char *data2 = GET_INPUT_DATA(pCtx, 1); TSKEY key1 = pCtx->ptsList[0]; TSKEY key2 = pCtx->ptsList[1]; @@ -4120,14 +3796,14 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - const char *input = GET_INPUT_CHAR(pCtx); + const char *input = GET_INPUT_DATA_LIST(pCtx); // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_CHAR_INDEX(pCtx, i); + char *d = GET_INPUT_DATA(pCtx, i); tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -4137,7 +3813,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { } static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4199,7 +3875,6 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { return resultVal; } - static bool rate_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { return false; @@ -4221,18 +3896,17 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return true; } - static void rate_function(SQLFunctionCtx *pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + int32_t notNullElems = 0; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { tscDebug("%p rate_function() index of null data:%d", pCtx, i); continue; @@ -4280,7 +3954,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { } static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } @@ -4288,8 +3962,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // NOTE: keep the intermediate result into the interResultBuf SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - + TSKEY *primaryKey = GET_TS_LIST(pCtx); + int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4321,42 +3995,6 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } - - -static void rate_func_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - tscDebug("rate_func_merge() size:%d", pCtx->size); - - SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; - char *indicator = pCtx->aInputElemBuf; - - assert(1 == pCtx->size); - - int32_t numOfNotNull = 0; - for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(SRateInfo)) { - SRateInfo *pInput = (SRateInfo *)indicator; - if (DATA_SET_FLAG != pInput->hasResult) { - continue; - } - - numOfNotNull++; - memcpy(pBuf, pInput, sizeof(SRateInfo)); - tscDebug("%p rate_func_merge() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64, - pCtx, pInput->isIRate, pInput->firstKey, pInput->lastKey, pInput->firstValue, pInput->lastValue, pInput->CorrectionValue); - } - - SET_VAL(pCtx, numOfNotNull, 1); - - if (numOfNotNull > 0) { - pBuf->hasResult = DATA_SET_FLAG; - } - - return; -} - - - static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -4365,12 +4003,10 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; - tscDebug("%p rate_func_second_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", + tscDebug("%p rate_func_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } - - static void rate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4394,14 +4030,13 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } - static void irate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; - + TSKEY *primaryKey = GET_TS_LIST(pCtx); + tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); if (pCtx->size < 1) { @@ -4409,7 +4044,7 @@ static void irate_function(SQLFunctionCtx *pCtx) { } for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_CHAR_INDEX(pCtx, i); + char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { tscDebug("%p irate_function() index of null data:%d", pCtx, i); continue; @@ -4452,15 +4087,15 @@ static void irate_function(SQLFunctionCtx *pCtx) { } static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { - void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); + void *pData = GET_INPUT_DATA(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = pCtx->ptsList; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); @@ -4490,7 +4125,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_CHAR(pCtx); + char * input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { SRateInfo *pInput = (SRateInfo *)input; @@ -4522,11 +4157,6 @@ static void sumrate_func_merge(SQLFunctionCtx *pCtx) { do_sumrate_merge(pCtx); } -static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { - tscDebug("%p sumrate_func_second_merge() process ...", pCtx); - do_sumrate_merge(pCtx); -} - static void sumrate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4590,7 +4220,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, count_func_merge, - count_func_merge, count_load_data_info, }, { @@ -4605,7 +4234,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, sum_func_merge, - sum_func_second_merge, statisRequired, }, { @@ -4620,7 +4248,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, avg_finalizer, avg_func_merge, - avg_func_second_merge, statisRequired, }, { @@ -4635,7 +4262,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, min_func_merge, - min_func_second_merge, statisRequired, }, { @@ -4650,7 +4276,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, max_func_merge, - max_func_second_merge, statisRequired, }, { @@ -4665,7 +4290,6 @@ SQLAggFuncElem aAggs[] = {{ stddev_next_step, stddev_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4680,7 +4304,6 @@ SQLAggFuncElem aAggs[] = {{ percentile_next_step, percentile_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4695,7 +4318,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, apercentile_finalizer, apercentile_func_merge, - apercentile_func_second_merge, dataBlockRequired, }, { @@ -4710,7 +4332,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, firstFuncRequired, }, { @@ -4725,7 +4346,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, noop1, - noop1, lastFuncRequired, }, { @@ -4740,8 +4360,7 @@ SQLAggFuncElem aAggs[] = {{ noop2, no_next_step, last_row_finalizer, - noop1, - last_dist_func_second_merge, + last_dist_func_merge, dataBlockRequired, }, { @@ -4757,7 +4376,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, top_func_merge, - top_func_second_merge, dataBlockRequired, }, { @@ -4773,7 +4391,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, top_bottom_func_finalizer, bottom_func_merge, - bottom_func_second_merge, dataBlockRequired, }, { @@ -4787,7 +4404,6 @@ SQLAggFuncElem aAggs[] = {{ spread_function_f, no_next_step, spread_function_finalizer, - spread_func_merge, spread_func_sec_merge, count_load_data_info, }, @@ -4802,7 +4418,6 @@ SQLAggFuncElem aAggs[] = {{ twa_function_f, no_next_step, twa_function_finalizer, - twa_func_merge, twa_function_copy, dataBlockRequired, }, @@ -4818,7 +4433,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, leastsquares_finalizer, noop1, - noop1, dataBlockRequired, }, { @@ -4833,7 +4447,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4848,7 +4461,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4863,7 +4475,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4878,7 +4489,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, ts_comp_finalize, copy_function, - copy_function, dataBlockRequired, }, { @@ -4893,7 +4503,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4908,7 +4517,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4923,7 +4531,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, no_data_info, }, { @@ -4938,7 +4545,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - copy_function, dataBlockRequired, }, { @@ -4953,7 +4559,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, noop1, - noop1, dataBlockRequired, }, // distributed version used in two-stage aggregation processes @@ -4969,7 +4574,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, first_dist_func_merge, - first_dist_func_second_merge, firstDistFuncRequired, }, { @@ -4984,7 +4588,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, function_finalizer, last_dist_func_merge, - last_dist_func_second_merge, lastDistFuncRequired, }, { @@ -4998,7 +4601,6 @@ SQLAggFuncElem aAggs[] = {{ do_sum_f, // todo filter handle no_next_step, doFinalizer, - noop1, copy_function, dataBlockRequired, }, @@ -5013,7 +4615,6 @@ SQLAggFuncElem aAggs[] = {{ rate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, @@ -5028,7 +4629,6 @@ SQLAggFuncElem aAggs[] = {{ irate_function_f, no_next_step, rate_finalizer, - rate_func_merge, rate_func_copy, dataBlockRequired, }, @@ -5044,7 +4644,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5059,7 +4658,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5074,7 +4672,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5089,7 +4686,6 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, sumrate_finalizer, sumrate_func_merge, - sumrate_func_second_merge, dataBlockRequired, }, { @@ -5104,6 +4700,5 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, noop1, noop1, - noop1, dataBlockRequired, }}; diff --git a/src/query/src/qSyntaxtreefunction.c b/src/query/src/qArithmeticOperator.c similarity index 99% rename from src/query/src/qSyntaxtreefunction.c rename to src/query/src/qArithmeticOperator.c index 7f7fca2c1e0ca6ce85f0df1c1b6e6dce82d8c000..0955d48df0bacbdf748c8a06f6a6364387d9be9d 100644 --- a/src/query/src/qSyntaxtreefunction.c +++ b/src/query/src/qArithmeticOperator.c @@ -15,7 +15,7 @@ #include "os.h" -#include "qSyntaxtreefunction.h" +#include "qArithmeticOperator.h" #include "taosdef.h" #include "tutil.h" @@ -1234,7 +1234,7 @@ _bi_consumer_fn_t rem_function_arraylist[8][10] = { //////////////////////////////////////////////////////////////////////////////////////////////////////////// -_bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t optr) { +_bi_consumer_fn_t getArithmeticOperatorFn(int32_t leftType, int32_t rightType, int32_t optr) { switch (optr) { case TSDB_BINARY_OP_ADD: return add_function_arraylist[leftType][rightType]; diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index a65f4a6dc95a586896372a648dc19ad2bb88b154..c23a79419660f0e474854e005db86a18971c0c96 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -16,29 +16,18 @@ #include "os.h" #include "exception.h" +#include "qArithmeticOperator.h" #include "qAst.h" -#include "qSyntaxtreefunction.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" #include "tname.h" +#include "tschemautil.h" #include "tsdb.h" #include "tskiplist.h" #include "tsqlfunction.h" -#include "tstoken.h" -#include "tschemautil.h" - -typedef struct { - char* v; - int32_t optr; -} SEndPoint; - -typedef struct { - SEndPoint* start; - SEndPoint* end; -} SQueryCond; static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { if (pLeft->nodeType == TSQL_NODE_COL) { @@ -53,323 +42,6 @@ static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, co } } -void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { - if (pNode == NULL) { - return; - } - - if (pNode->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&pNode, fp); - } else if (pNode->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy(pNode->pVal); - } else if (pNode->nodeType == TSQL_NODE_COL) { - free(pNode->pSchema); - } - - free(pNode); -} - -void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { - if (*pExpr == NULL) { - return; - } - - if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); - tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); - - if (fp != NULL) { - fp((*pExpr)->_node.info); - } - } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { - tVariantDestroy((*pExpr)->pVal); - free((*pExpr)->pVal); - } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { - free((*pExpr)->pSchema); - } - - free(*pExpr); - *pExpr = NULL; -} - -// todo check for malloc failure -static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { - int32_t optr = queryColInfo->optr; - - if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || - optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { - pCond->start = calloc(1, sizeof(SEndPoint)); - pCond->start->optr = queryColInfo->optr; - pCond->start->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - pCond->end = calloc(1, sizeof(SEndPoint)); - pCond->end->optr = queryColInfo->optr; - pCond->end->v = queryColInfo->q; - } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - -static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { - SSkipListIterator* iter = NULL; - - SQueryCond cond = {0}; - if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { - //todo handle error - } - - if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); - } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); - } - - if (cond.start != NULL) { - int32_t optr = cond.start->optr; - - if (optr == TSDB_RELATION_EQUAL) { // equals - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - if (ret != 0) { - break; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal - bool comp = true; - int32_t ret = 0; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); - assert(ret >= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_GREATER) { - continue; - } else { - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; - } - } - } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal - bool comp = true; - - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - tSkipListDestroyIter(iter); - - comp = true; - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); - while(tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { - continue; - } - - STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - - } else { - assert(0); - } - } else { - int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; - if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { - bool comp = true; - int32_t ret = 0; - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - if (comp) { - ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); - assert(ret <= 0); - } - - if (ret == 0 && optr == TSDB_RELATION_LESS) { - continue; - } else { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - comp = false; // no need to compare anymore - } - } - } else { - assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - - bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); - if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || - (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { - STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(result, &info); - } - } - } - } - - free(cond.start); - free(cond.end); - tSkipListDestroyIter(iter); -} - -static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - //non-leaf nodes, recursively traverse the expression tree in the post-root order - if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { - if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - if (filterItem(pLeft, pItem, param)) { - return true; - } - - // left child does not satisfy the query condition, try right child - return filterItem(pRight, pItem, param); - } else { // and - if (!filterItem(pLeft, pItem, param)) { - return false; - } - - return filterItem(pRight, pItem, param); - } - } - - // handle the leaf node - param->setupInfoFn(pExpr, param->pExtInfo); - return param->nodeFilterFn(pItem, pExpr->_node.info); -} - -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { - taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); - } - } - tSkipListDestroyIter(iter); -} - -static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { - SSkipListIterator* iter = tSkipListCreateIter(pSkipList); - - while (tSkipListIterNext(iter)) { - bool addToResult = false; - - SSkipListNode *pNode = tSkipListIterGet(iter); - char * pData = SL_GET_NODE_DATA(pNode); - - tstr *name = (tstr*) tsdbGetTableName((void*) pData); - - // todo speed up by using hash - if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { - if (pQueryInfo->optr == TSDB_RELATION_IN) { - addToResult = pQueryInfo->compare(name, pQueryInfo->q); - } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { - addToResult = !pQueryInfo->compare(name, pQueryInfo->q); - } - } else { - addToResult = filterFp(pNode, pQueryInfo); - } - - if (addToResult) { - STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(res, &info); - } - } - - tSkipListDestroyIter(iter); -} - -// post-root order traverse syntax tree -void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { - if (pExpr == NULL) { - return; - } - - tExprNode *pLeft = pExpr->_node.pLeft; - tExprNode *pRight = pExpr->_node.pRight; - - // column project - if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { - assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); - - param->setupInfoFn(pExpr, param->pExtInfo); - - tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { - tQueryIndexColumn(pSkipList, pQueryInfo, result); - } else { - tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); - } - - return; - } - - // The value of hasPK is always 0. - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; - assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); - - //apply the hierarchical expression to every node in skiplist for find the qualified nodes - tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - -#if 0 - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } - - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); - - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); -#endif -} - static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: { @@ -430,7 +102,73 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf } } -void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, +void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { + if (pNode == NULL) { + return; + } + + if (pNode->nodeType == TSQL_NODE_EXPR) { + tExprTreeDestroy(&pNode, fp); + } else if (pNode->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy(pNode->pVal); + } else if (pNode->nodeType == TSQL_NODE_COL) { + free(pNode->pSchema); + } + + free(pNode); +} + +void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { + if (*pExpr == NULL) { + return; + } + + if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { + tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); + tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); + + if (fp != NULL) { + fp((*pExpr)->_node.info); + } + } else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) { + tVariantDestroy((*pExpr)->pVal); + free((*pExpr)->pVal); + } else if ((*pExpr)->nodeType == TSQL_NODE_COL) { + free((*pExpr)->pSchema); + } + + free(*pExpr); + *pExpr = NULL; +} + +bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) { + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + //non-leaf nodes, recursively traverse the expression tree in the post-root order + if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) { + if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + if (exprTreeApplayFilter(pLeft, pItem, param)) { + return true; + } + + // left child does not satisfy the query condition, try right child + return exprTreeApplayFilter(pRight, pItem, param); + } else { // and + if (!exprTreeApplayFilter(pLeft, pItem, param)) { + return false; + } + + return exprTreeApplayFilter(pRight, pItem, param); + } + } + + // handle the leaf node + param->setupInfoFn(pExpr, param->pExtInfo); + return param->nodeFilterFn(pItem, pExpr->_node.info); +} + +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { return; @@ -442,7 +180,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, /* the left output has result from the left child syntax tree */ char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); if (pLeft->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); } /* the right output has result from the right child syntax tree */ @@ -450,7 +188,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, char *pdata = malloc(sizeof(int64_t) * numOfRows); if (pRight->nodeType == TSQL_NODE_EXPR) { - tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); + arithmeticTreeTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } if (pLeft->nodeType == TSQL_NODE_EXPR) { @@ -459,11 +197,11 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, * exprLeft + exprRight * the type of returned value of one expression is always double float precious */ - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); // set input buffer char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); @@ -475,14 +213,14 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); } } else if (pLeft->nodeType == TSQL_NODE_COL) { // column data specified on left-hand-side char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId); if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); @@ -494,12 +232,12 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); // both columns are descending order, do not reverse the source data fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); @@ -511,13 +249,13 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } else { // column data specified on left-hand-side if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); if (order == TSDB_ORDER_DESC) { reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows); @@ -527,7 +265,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12 - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); + _bi_consumer_fn_t fp = getArithmeticOperatorFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, TSDB_ORDER_ASC); } } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0d3b068a4aad6c858cf63c7b6537bf98b7279352..be4d849fb99e1f030f48f08e43e7407cd3676451 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -53,7 +53,7 @@ #define TIME_WINDOW_COPY(_dst, _src) do {\ (_dst).skey = (_src).skey;\ (_dst).ekey = (_src).ekey;\ -} while (0); +} while (0) enum { // when query starts to execute, this status will set @@ -86,7 +86,6 @@ typedef struct { int32_t status; // query status TSKEY lastKey; // the lastKey value before query executed STimeWindow w; // whole query time window - STimeWindow curWindow; // current query window int32_t windowIndex; // index of active time window result for interval query STSCursor cur; } SQueryStatusInfo; @@ -178,11 +177,10 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { #define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) // todo move to utility -static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); +static int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo); static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); static void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); -static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, @@ -195,7 +193,6 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); @@ -291,7 +288,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static int32_t getMergeResultGroupId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getMergeResultGroupId(int32_t groupIndex) { int32_t base = 50000000; return base + (groupIndex * 10000); } @@ -466,16 +463,34 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { + bool existed = false; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - int32_t *p1 = - (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (p1 != NULL) { - pResultRowInfo->curIndex = *p1; + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; + } + + if (p1 != NULL) { + for(int32_t i = pResultRowInfo->size - 1; i >= 0; --i) { + if (pResultRowInfo->pResult[i] == (*p1)) { + pResultRowInfo->curIndex = i; + existed = true; + break; + } + } + } } else { - if (!masterscan) { // not master scan, do not add new timewindow - return NULL; + if (p1 != NULL) { // group by column query + return *p1; } + } + if (!existed) { // TODO refactor // more than the capacity, reallocate the resources if (pResultRowInfo->size >= pResultRowInfo->capacity) { @@ -499,17 +514,23 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes pResultRowInfo->capacity = (int32_t)newCapacity; } - SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool); - pResultRowInfo->pResult[pResultRowInfo->size] = pResult; - int32_t ret = initResultRow(pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + SResultRow *pResult = NULL; + + if (p1 == NULL) { + pResult = getNewResultRow(pRuntimeEnv->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + } else { + pResult = *p1; } - // add a new result set for a new group + pResultRowInfo->pResult[pResultRowInfo->size] = pResult; pResultRowInfo->curIndex = pResultRowInfo->size++; - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), - (char *)&pResultRowInfo->curIndex, sizeof(int32_t)); } // too many time window in query @@ -591,7 +612,6 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf if (pData->num >= numOfRowsPerPage) { // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer @@ -614,24 +634,20 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SDataBlockInfo* pBockInfo, - STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) { +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, + bool masterscan, SResultRow** pResult, int64_t groupId) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - // todo refactor - int64_t uid = getResultInfoUId(pRuntimeEnv); - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, uid); + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); if (pResultRow == NULL) { - *newWind = false; - return masterscan? -1:0; // no master scan, no result generated means error occurs + *pResult = NULL; + return TSDB_CODE_SUCCESS; } - *newWind = true; - // not assign result buffer yet, add new result buffer if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -701,81 +717,60 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -static int32_t updateResultRowCurrentIndex(SResultRowInfo* pWindowResInfo, TSKEY lastKey, bool ascQuery) { - int32_t i = 0; +static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; - - int32_t numOfClosed = 0; - for (i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pResult = pWindowResInfo->pResult[i]; + int32_t i = 0; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; if (pResult->closed) { - numOfClosed += 1; - continue; + break; } - TSKEY ekey = pResult->win.ekey; - if ((ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { - closeResultRow(pWindowResInfo, i); + // new closed result rows + if (timeWindowInterpo) { + if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { + if (i > 0) { // the first time window, the startInterp is false. + assert(pResult->startInterp); + } + + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } } else { - skey = pResult->win.skey; - break; + if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } } } - // all windows are closed, set the last one to be the skey + // all result rows are closed, set the last one to be the skey if (skey == TSKEY_INITIAL_VAL) { - assert(i == pWindowResInfo->size); - pWindowResInfo->curIndex = pWindowResInfo->size - 1; + pResultRowInfo->curIndex = pResultRowInfo->size - 1; } else { - pWindowResInfo->curIndex = i; - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; - } - - return numOfClosed; -} - -/** - * NOTE: the query status only set for the first scan of master scan. - */ -static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->scanFlag != MASTER_SCAN || pWindowResInfo->size == 0) { - return pWindowResInfo->size; - } - - // no qualified results exist, abort check - int32_t numOfClosed = 0; - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - - // query completed - if ((lastKey >= pQuery->current->win.ekey && ascQuery) || (lastKey <= pQuery->current->win.ekey && (!ascQuery))) { - closeAllResultRows(pWindowResInfo); - - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); - } else { // set the current index to be the last unclosed window - numOfClosed = updateResultRowCurrentIndex(pWindowResInfo, lastKey, ascQuery); - - // the number of completed slots are larger than the threshold, return current generated results to client. - if (numOfClosed > pQuery->rec.threshold) { - qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", - GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - } else { - qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, - numOfClosed); + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } } - } - // output has reached the limitation, set query completed - if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && - pRuntimeEnv->scanFlag == MASTER_SCAN) { - setQueryStatus(pQuery, QUERY_COMPLETED); + pResultRowInfo->curIndex = i + 1; // current not closed result object + pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; } +} - assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); - return numOfClosed; +static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery, bool timeWindowInterpo) { + if ((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey && (!ascQuery))) { + closeAllResultRows(pResultRowInfo); + pResultRowInfo->curIndex = pResultRowInfo->size - 1; + } else { + int32_t step = ascQuery? 1:-1; + doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey - step, ascQuery, timeWindowInterpo); + } } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, @@ -818,52 +813,44 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, - int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { - SQuery * pQuery = pRuntimeEnv->pQuery; +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { + SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool hasPrev = pCtx[0].preAggVals.isSet; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; - pCtx[k].size = forwardStep; - pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); - - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - pCtx[k].ptsList = &tsCol[pCtx[k].startOffset]; - } + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; + pCtx[k].size = forwardStep; + pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { - pCtx[k].preAggVals.isSet = false; - } + int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunction(&pCtx[k]); - } + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].preAggVals.isSet && forwardStep < numOfTotal) { + pCtx[k].preAggVals.isSet = false; + } - // restore it - pCtx[k].preAggVals.isSet = hasPrev; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunction(&pCtx[k]); } + + // restore it + pCtx[k].preAggVals.isSet = hasPrev; } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { - SQuery * pQuery = pRuntimeEnv->pQuery; +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset) { + SQuery *pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pCtx[k].nStartQueryTimestamp = pWin->skey; - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pCtx[k], offset); } } } @@ -975,8 +962,7 @@ static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { return NULL; } -static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, - SArray *pDataBlock) { +static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { if (pDataBlock == NULL) { return NULL; } @@ -987,15 +973,9 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas int32_t functionId = pQuery->pExpr1[col].base.functionId; if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - - sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); - sas->colList = pQuery->colList; - sas->numOfCols = pQuery->numOfCols; - sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); - - if (sas->data == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); + sas->colList = pQuery->colList; + sas->numOfCols = pQuery->numOfCols; // here the pQuery->colList and sas->colList are identical int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); @@ -1098,13 +1078,13 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32 return true; } -static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) { +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, + int32_t rowIndex) { if (pDataBlock == NULL) { return; } SQuery* pQuery = pRuntimeEnv->pQuery; - int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); @@ -1124,6 +1104,45 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY return ts; } +static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray *pDataBlock, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + if (!pRuntimeEnv->timeWindowInterpo) { + return; + } + + assert(pDataBlock != NULL); + + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); + + TSKEY *tsCols = (TSKEY *)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } + + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + } +} + /** * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions * @param pRuntimeEnv @@ -1139,22 +1158,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + TSKEY *tsCols = NULL; if (pDataBlock != NULL) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); tsCols = (TSKEY *)(pColInfo->pData); } - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -1164,80 +1180,50 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { - tfree(sasArray); - return; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + goto _end; } int32_t forwardStep = 0; int32_t startPos = pQuery->pos; - // in case of repeat scan/reverse scan, no new time window added. - if (hasTimeWindow) { - TSKEY ekey = reviseWindowEkey(pQuery, &win); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - - // prev time window not interpolation yet. - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { - for(int32_t j = prevIndex; j < curIndex; ++j) { - SResultRow *pRes = pWindowResInfo->pResult[j]; + TSKEY ekey = reviseWindowEkey(pQuery, &win); + forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows); + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { + for(int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow *pRes = pWindowResInfo->pResult[j]; + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; } - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - assert (ret == TSDB_CODE_SUCCESS); // null data, too many state code - } + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &w, masterScan, &pResult, groupId); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - // window start key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = pQuery->pos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } - - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step; + int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - } + doBlockwiseApplyFunctions(pRuntimeEnv, &w, startPos, 0, tsCols, pDataBlockInfo->rows); } - bool pStatus = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + assert (ret == TSDB_CODE_SUCCESS); } - int32_t index = pWindowResInfo->curIndex; - STimeWindow nextWin = win; + // window start key interpolation + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &win, pQuery->pos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos); @@ -1246,50 +1232,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (!hasTimeWindow) { - continue; - } - - TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); + ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); // window start(end) key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } - - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1)*step; - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - } - } - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &nextWin, startPos, forwardStep); + doBlockwiseApplyFunctions(pRuntimeEnv, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } - pWindowResInfo->curIndex = index; } else { /* * the sqlfunctionCtx parameters should be set done before all functions are invoked, @@ -1305,28 +1260,14 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } } + _end: if (pRuntimeEnv->timeWindowInterpo) { - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock); - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); + int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, rowIndex); } - - tfree(sasArray); } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - if (isNull(pData, type)) { // ignore the null value - return -1; - } - - int32_t GROUPRESULTID = 1; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; // not assign result buffer yet, add new result buffer, TODO remove it @@ -1338,16 +1279,12 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); - - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - - uint64_t uid = groupIndex; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); - if (pResultRow == NULL) { return -1; } + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, groupIndex); + assert (pResultRow != NULL); + int64_t v = -1; GET_TYPED_DATA(v, int64_t, type, pData); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { @@ -1363,7 +1300,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; } @@ -1498,7 +1435,7 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]); + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); } else { GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } @@ -1558,22 +1495,19 @@ static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pData } static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* item = pQuery->current; + int64_t groupId = item->groupIndex; + SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0); TSKEY *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL; - bool groupbyColumnValue = pRuntimeEnv->groupbyNormalCol; - - SArithmeticSupport *sasArray = calloc((size_t)pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (sasArray == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + bool groupbyColumnValue = pRuntimeEnv->groupbyColumn; int16_t type = 0; int16_t bytes = 0; @@ -1585,8 +1519,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); pCtx[k].size = 1; } @@ -1631,19 +1565,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); - int64_t ts = tsCols[offset]; + int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; - } - - if (!hasTimeWindow) { - continue; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { // null data, too many state code + goto _end; } // window start key interpolation @@ -1653,19 +1582,20 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { for (int32_t k = prevWindowIndex; k < curIndex; ++k) { SResultRow *pRes = pWindowResInfo->pResult[k]; + if (pRes->closed) { + assert(resultRowInterpolated(pResult, RESULT_ROW_START_INTERP) && resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + continue; + } - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &pRes->win, masterScan, &pResult, groupId); assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); - - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &pRes->win, offset); } // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, - &pResult); + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1674,12 +1604,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); } - bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); + doRowwiseApplyFunctions(pRuntimeEnv, &win, offset); + int32_t index = pWindowResInfo->curIndex; STimeWindow nextWin = win; - int32_t index = pWindowResInfo->curIndex; - while (1) { getNextTimeWindow(pQuery, &nextWin); if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || @@ -1692,27 +1620,28 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } // null data, failed to allocate more memory buffer - hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) { + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { break; } - if (hasTimeWindow) { - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); - closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); - } + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); + doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset); } + // restore the index, add the result row will move the index pWindowResInfo->curIndex = index; } else { // other queries // decide which group this rows belongs to according to current state value if (groupbyColumnValue) { char *val = groupbyColumnData + bytes * offset; + if (isNull(val, type)) { // ignore the null value + continue; + } int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } } @@ -1736,55 +1665,47 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } - assert(offset >= 0); - if (tsCols != NULL) { - item->lastKey = tsCols[offset] + step; - } else { - item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step; + _end: + assert(offset >= 0 && tsCols != NULL); + if (prevTs != INT64_MIN) { + assert(prevRowIndex >= 0); + item->lastKey = prevTs + step; } - if (pRuntimeEnv->pTsBuf != NULL) { - item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + // In case of all rows in current block are not qualified + if (pRuntimeEnv->timeWindowInterpo && prevRowIndex != -1) { + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, prevRowIndex); } - // todo refactor: extract method - for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { - if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { - continue; - } - - tfree(sasArray[i].data); + if (pRuntimeEnv->pTsBuf != NULL) { + item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); } - - free(sasArray); } static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQInfo = pQuery->current; - SResultRowInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + STableQueryInfo* pTableQueryInfo = pQuery->current; + SResultRowInfo* pResultRowInfo = &pRuntimeEnv->windowResInfo; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } - // update the lastkey of current table + // update the lastkey of current table for projection/aggregation query TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // interval query with limit applied int32_t numOfRes = 0; - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo); - } else if (pRuntimeEnv->groupbyNormalCol) { - closeAllResultRows(pWindowResInfo); - numOfRes = pWindowResInfo->size; + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { + numOfRes = pResultRowInfo->size; + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); } else { // projection query - numOfRes = (int32_t)getNumOfResult(pRuntimeEnv); + numOfRes = (int32_t) getNumOfResult(pRuntimeEnv); // update the number of output result if (numOfRes > 0 && pQuery->checkBuffer == 1) { @@ -1799,8 +1720,8 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl setQueryStatus(pQuery, QUERY_COMPLETED); } - if (((pTableQInfo->lastKey > pTableQInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pTableQInfo->lastKey < pTableQInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { + if (((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { setQueryStatus(pQuery, QUERY_COMPLETED); } } @@ -1838,7 +1759,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY uint32_t status = aAggs[functionId].nStatus; if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = &tsCol[pCtx->startOffset]; + pCtx->ptsList = tsCol; } if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { @@ -1956,8 +1877,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); pRuntimeEnv->offset = calloc(pQuery->numOfOutput, sizeof(int16_t)); pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->sasArray = calloc(pQuery->numOfOutput, sizeof(SArithmeticSupport)); - if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { + if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL || pRuntimeEnv->sasArray == NULL) { goto _clean; } @@ -2033,18 +1955,24 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->param[1].i64Key = pQuery->order.orderColId; } + if (functionId == TSDB_FUNC_ARITHM) { + pRuntimeEnv->sasArray[i].data = calloc(pQuery->numOfCols, POINTER_BYTES); + if (pRuntimeEnv->sasArray[i].data == NULL) { + goto _clean; + } + } + if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes; } - } *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; // if it is group by normal column, do not set output buffer, the output buffer is pResult // fixed output query/multi-output query for normal table - if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { + if (!pRuntimeEnv->groupbyColumn && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { resetDefaultResInfoOutputBuf(pRuntimeEnv); } @@ -2059,6 +1987,7 @@ _clean: tfree(pRuntimeEnv->pCtx); tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->rowCellInfoOffset); + tfree(pRuntimeEnv->sasArray); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -2102,6 +2031,14 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->pCtx); } + if (pRuntimeEnv->sasArray != NULL) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + tfree(pRuntimeEnv->sasArray[i].data); + } + + tfree(pRuntimeEnv->sasArray); + } + pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); @@ -2154,7 +2091,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { } // Note:top/bottom query is fixed output query - if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyColumn) { return true; } @@ -2182,8 +2119,8 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { // todo refactor with isLastRowQuery bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionID = pQuery->pExpr1[i].base.functionId; - if (functionID == TSDB_FUNC_INTERP) { + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_INTERP) { return true; } } @@ -2618,6 +2555,8 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW *status = BLK_DATA_NO_NEEDED; SQuery *pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + SQueryCostInfo* pCost = &pRuntimeEnv->summary; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) { @@ -2634,15 +2573,13 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - bool hasTimeWindow = false; SResultRow* pResult = NULL; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow, &pResult) != - TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } } @@ -2795,7 +2732,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { + if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyColumn && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { @@ -2840,13 +2777,9 @@ static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo if (QUERY_IS_ASC_QUERY(pQuery)) { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } } @@ -2916,13 +2849,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { setQueryStatus(pQuery, QUERY_COMPLETED); } - if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - closeAllResultRows(&pRuntimeEnv->windowResInfo); - pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window - } else { - assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - } + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + closeAllResultRows(&pRuntimeEnv->windowResInfo); + pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window } return 0; @@ -3027,50 +2956,6 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { } } -static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { - SQuery * pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (!mergeFlag) { - pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; - pCtx[i].currentStage = FIRST_STAGE_MERGE; - - RESET_RESULT_INFO(pCtx[i].resultInfo); - aAggs[functionId].init(&pCtx[i]); - } - - pCtx[i].hasNull = true; - pCtx[i].nStartQueryTimestamp = timestamp; - pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); - - // in case of tag column, the tag information should be extracted from input buffer - if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { - tVariantDestroy(&pCtx[i].tag); - - int32_t type = pCtx[i].outputType; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); - } else { - tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); - } - - } - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TAG_DUMMY) { - continue; - } - - aAggs[functionId].distMergeFunc(&pCtx[i]); - } -} - static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { switch (srcDataType) { @@ -3176,19 +3061,18 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim typedef struct SCompSupporter { STableQueryInfo **pTableQueryInfo; - int32_t * position; - SQInfo * pQInfo; + int32_t *rowIndex; + int32_t order; } SCompSupporter; int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { - int32_t left = *(int32_t *)pLeft; + int32_t left = *(int32_t *)pLeft; int32_t right = *(int32_t *)pRight; SCompSupporter * supporter = (SCompSupporter *)param; - SQueryRuntimeEnv *pRuntimeEnv = &supporter->pQInfo->runtimeEnv; - int32_t leftPos = supporter->position[left]; - int32_t rightPos = supporter->position[right]; + int32_t leftPos = supporter->rowIndex[left]; + int32_t rightPos = supporter->rowIndex[right]; /* left source is exhausted */ if (leftPos == -1) { @@ -3200,53 +3084,55 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return -1; } - SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; - SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); - tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); + STableQueryInfo** pList = supporter->pTableQueryInfo; - char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); - TSKEY leftTimestamp = GET_INT64_VAL(b1); + SResultRowInfo *pWindowResInfo1 = &(pList[left]->windowResInfo); + SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); + TSKEY leftTimestamp = pWindowRes1->win.skey; - SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; + SResultRowInfo *pWindowResInfo2 = &(pList[right]->windowResInfo); SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); - tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); - - char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); - TSKEY rightTimestamp = GET_INT64_VAL(b2); + TSKEY rightTimestamp = pWindowRes2->win.skey; if (leftTimestamp == rightTimestamp) { return 0; } - return leftTimestamp > rightTimestamp ? 1 : -1; + if (supporter->order == TSDB_ORDER_ASC) { + return (leftTimestamp > rightTimestamp)? 1:-1; + } else { + return (leftTimestamp < rightTimestamp)? 1:-1; + } } -int32_t mergeIntoGroupResult(SQInfo *pQInfo) { +int32_t mergeGroupResult(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); - int32_t ret = TSDB_CODE_SUCCESS; - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); while (pQInfo->groupIndex < numOfGroups) { SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); - ret = mergeIntoGroupResultImpl(pQInfo, group); - if (ret < 0) { // not enough disk space to save the data into disk - return -1; - } - pQInfo->groupIndex += 1; + int32_t ret = mergeIntoGroupResultImpl(pGroupResInfo, group, pQInfo); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } // this group generates at least one result, return results - if (ret > 0) { + pQInfo->groupIndex += 1; + if (taosArrayGetSize(pGroupResInfo->pRows) > 0) { break; } - assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); + taosArrayClear(pGroupResInfo->pRows); + + pGroupResInfo->index = 0; + pGroupResInfo->rowId = 0; } - SGroupResInfo* info = &pQInfo->groupResInfo; - if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) { + if (pQInfo->groupIndex == numOfGroups && taosArrayGetSize(pGroupResInfo->pRows) == 0) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -3258,89 +3144,28 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { return TSDB_CODE_SUCCESS; } +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t* index, int32_t orderType); + void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - // all results have been return to client, try next group - if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) { - pGroupResInfo->numOfDataPages = 0; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; - + // all results in current group have been returned to client, try next group + if (pGroupResInfo->index >= taosArrayGetSize(pGroupResInfo->pRows)) { // current results of group has been sent to client, try next group - if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { + if (mergeGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { return; // failed to save data in the disk } // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (taosArrayGetSize(pGroupResInfo->pRows) == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } } - SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - int32_t id = pQInfo->groupResInfo.groupId; - SIDList list = getDataBufPagesIdList(pResultBuf, id); - - int32_t offset = 0; - int32_t numOfCopiedRows = 0; - - size_t size = taosArrayGetSize(list); - assert(size == pGroupResInfo->numOfDataPages); - - bool done = false; - - //TODO add API for release none-dirty pages -// SPageInfo* prev = NULL; - - for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { - SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); - - // release previous buffer pages -// if (prev == NULL) { -// prev = pi; -// } else { -// if (prev->pageId != pi->pageId) { -// releaseResBufPageInfo(pResultBuf, prev); -// prev = pi; -// } -// } - - assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); - int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); - - if (numOfRes > pQuery->rec.capacity - offset) { - numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); - pGroupResInfo->rowId += numOfCopiedRows; - done = true; - } else { - numOfCopiedRows = (int32_t)pData->num; - - pGroupResInfo->pageId += 1; - pGroupResInfo->rowId = 0; - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - char * pDest = pQuery->sdata[i]->data; - - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, - (size_t)bytes * numOfCopiedRows); - } - - offset += numOfCopiedRows; - if (done) { - break; - } - } - - assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; + int32_t size = (int32_t) taosArrayGetSize(pGroupResInfo->pRows); + pQuery->rec.rows = doCopyToSData(pQInfo, pGroupResInfo->pRows->pData, (int32_t) size, &pGroupResInfo->index, TSDB_ORDER_ASC); } int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { @@ -3368,155 +3193,99 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResu return 0; } -int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { +int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery); - size_t size = taosArrayGetSize(pGroup); - tFilePage **buffer = pQuery->sdata; + int32_t code = TSDB_CODE_SUCCESS; - int32_t *posList = calloc(size, sizeof(int32_t)); - STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); + int32_t *posList = NULL; + SLoserTreeInfo *pTree = NULL; + STableQueryInfo **pTableQueryInfoList = NULL; + + size_t size = taosArrayGetSize(pTableList); + if (pGroupResInfo->pRows == NULL) { + pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); + } - if (pTableList == NULL || posList == NULL) { - tfree(posList); - tfree(pTableList); + posList = calloc(size, sizeof(int32_t)); + pTableQueryInfoList = malloc(POINTER_BYTES * size); + if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL) { qError("QInfo:%p failed alloc memory", pQInfo); - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; } - // todo opt for the case of one table per group int32_t numOfTables = 0; - SIDList pageList = NULL; - int32_t tid = -1; - for (int32_t i = 0; i < size; ++i) { - STableQueryInfo *item = taosArrayGetP(pGroup, i); - - SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); - if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables++] = item; - tid = TSDB_TABLEID(item->pTable)->tid; - pageList = list; + STableQueryInfo *item = taosArrayGetP(pTableList, i); + if (item->windowResInfo.size > 0) { + pTableQueryInfoList[numOfTables++] = item; } } // there is no data in current group + // no need to merge results since only one table in each group if (numOfTables == 0) { - tfree(posList); - tfree(pTableList); - return 0; - } else if (numOfTables == 1) { // no need to merge results since only one table in each group - tfree(posList); - tfree(pTableList); - - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); - pGroupResInfo->groupId = tid; - pGroupResInfo->pageId = 0; - pGroupResInfo->rowId = 0; - - return pGroupResInfo->numOfDataPages; + goto _end; } - SCompSupporter cs = {pTableList, posList, pQInfo}; + SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQuery->order.order}; - SLoserTreeInfo *pTree = NULL; - tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - - SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - - pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex); + int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); + if (ret != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; + } - // todo add windowRes iterator - int64_t lastTimestamp = -1; + int64_t lastTimestamp = ascQuery? INT64_MIN:INT64_MAX; int64_t startt = taosGetTimestampMs(); while (1) { if (isQueryKilled(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - - tfree(pTableList); - tfree(posList); - tfree(pTree); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + code = TSDB_CODE_TSC_QUERY_CANCELLED; + goto _end; } - int32_t pos = pTree->pNode[0].index; - - SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); + int32_t tableIndex = pTree->pNode[0].index; - char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); - TSKEY ts = GET_INT64_VAL(b); + SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->windowResInfo; + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); - assert(ts == pWindowRes->win.skey); int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); if (num <= 0) { - cs.position[pos] += 1; + cs.rowIndex[tableIndex] += 1; - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; - - // all input sources are exhausted - if (--numOfTables == 0) { + if (cs.rowIndex[tableIndex] >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; + if (--numOfTables == 0) { // all input sources are exhausted break; } } } else { - if (ts == lastTimestamp) { // merge with the last one - doMerge(pRuntimeEnv, ts, pWindowRes, true); - } else { // copy data to disk buffer - if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - return -1; - } - - resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - } + assert((pWindowRes->win.skey >= lastTimestamp && ascQuery) || (pWindowRes->win.skey <= lastTimestamp && !ascQuery)); - doMerge(pRuntimeEnv, ts, pWindowRes, false); - buffer[0]->num += 1; + if (pWindowRes->win.skey != lastTimestamp) { + taosArrayPush(pGroupResInfo->pRows, &pWindowRes); + pWindowRes->numOfRows = (uint32_t) num; } - lastTimestamp = ts; - - // move to the next element of current entry - int32_t currentPageId = pWindowRes->pageId; + lastTimestamp = pWindowRes->win.skey; - cs.position[pos] += 1; - if (cs.position[pos] >= pWindowResInfo->size) { - cs.position[pos] = -1; + // move to the next row of current entry + if ((++cs.rowIndex[tableIndex]) >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; // all input sources are exhausted - if (--numOfTables == 0) { + if ((--numOfTables) == 0) { break; } - } else { - // current page is not needed anymore - SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); - if (pNextWindowRes->pageId != currentPageId) { - releaseResBufPage(pRuntimeEnv->pResultBuf, page); - } } } - tLoserTreeAdjust(pTree, pos + pTree->numOfEntries); - } - - if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { - qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - - tfree(pTree); - tfree(pTableList); - tfree(posList); - return -1; - } + tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries); } int64_t endt = taosGetTimestampMs(); @@ -3527,65 +3296,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - tfree(pTableList); + _end: + tfree(pTableQueryInfoList); tfree(posList); tfree(pTree); -// tfree(pResultInfo); -// tfree(buf); - - return pQInfo->groupResInfo.numOfDataPages; -} - -int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - - // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. - int32_t pageId = -1; - int32_t capacity = pResultBuf->numOfRowsPerPage; - - int32_t remain = (int32_t) pQuery->sdata[0]->num; - int32_t offset = 0; - - while (remain > 0) { - int32_t rows = (remain > capacity)? capacity:remain; - assert(rows > 0); - - // get the output buffer page - tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); - buf->num = rows; - - // pagewisely copy to dest buffer - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - - char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; - char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; - memcpy(output, src, (size_t)(buf->num * bytes)); - } - - offset += rows; - remain -= rows; - - pGroupResInfo->numOfDataPages += 1; - } - - return TSDB_CODE_SUCCESS; -} - -void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; - pCtx[k].size = 1; - pCtx[k].startOffset = 0; - pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k); - - pQuery->sdata[k]->num = 0; - } + return code; } static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { @@ -3593,14 +3309,6 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * return; } - // order has changed already - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { - // do nothing, no results - } else {// NOTE: even win.skey != lastKey, the results may not generated. - pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; - } - SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; @@ -3644,7 +3352,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { // group by normal columns and interval query on normal table SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor @@ -3706,7 +3414,7 @@ void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; int32_t tid = 0; - int64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -3832,18 +3540,14 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; bool toContinue = false; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pResult = getResultRow(pWindowResInfo, i); - if (!pResult->closed) { - continue; - } setResultOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pExpr1[j].base.functionId; if (functId == TSDB_FUNC_TS) { @@ -3887,10 +3591,6 @@ static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY }; TIME_WINDOW_COPY(info.w, pQuery->window); - TIME_WINDOW_COPY(info.curWindow, pTableQueryInfo->win); - - info.curWindow.skey = start; - return info; } @@ -3906,9 +3606,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI } // reverse order time range - pQuery->window = pStatus->curWindow; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -3974,9 +3672,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { // store the start query position SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv, start); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); while (1) { doScanAllDataBlocks(pRuntimeEnv); @@ -3985,13 +3681,9 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan - if (qstatus.lastKey != pTableQueryInfo->lastKey) { - qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; - } else { // the lastkey does not increase, which means no data checked yet + if (qstatus.lastKey == pTableQueryInfo->lastKey) { qDebug("QInfo:%p no results generated in this scan", pQInfo); } - - qstatus.lastKey = pTableQueryInfo->lastKey; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { @@ -4007,7 +3699,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &qstatus.curWindow); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { @@ -4043,10 +3735,10 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->groupbyColumn) { closeAllResultRows(pWindowResInfo); } @@ -4100,7 +3792,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void pTableQueryInfo->cur.vgroupIndex = -1; // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { int32_t initialSize = 128; int32_t code = initResultRowInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { @@ -4127,9 +3819,9 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { * @param pDataBlockInfo */ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; @@ -4142,12 +3834,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - uint64_t uid = getResultInfoUId(pRuntimeEnv); + int64_t uid = 0; SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); - if (pResultRow == NULL) { - return; - } + assert (pResultRow != NULL); /* * not assign result buffer yet, add new result buffer @@ -4293,7 +3983,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { /** * In handling the both ascending and descending order super table query, we need to find the first qualified * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional * operations involve. */ STimeWindow w = TSWINDOW_INITIALIZER; @@ -4302,7 +3992,6 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { TSKEY sk = MIN(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey); getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); - pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { if (!QUERY_IS_ASC_QUERY(pQuery)) { @@ -4341,36 +4030,33 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) { +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t *index, int32_t orderType) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; int32_t numOfResult = 0; - int32_t startIdx = 0; + int32_t start = 0; int32_t step = -1; qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); - int32_t totalSet = numOfClosedResultRows(pResultInfo); - SResultRow** result = pResultInfo->pResult; - if (orderType == TSDB_ORDER_ASC) { - startIdx = pQInfo->groupIndex; + start = (*index); step = 1; } else { // desc order copy all data - startIdx = totalSet - pQInfo->groupIndex - 1; + start = numOfRows - (*index) - 1; step = -1; } SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { - if (result[i]->numOfRows == 0) { - pQInfo->groupIndex += 1; + for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { + if (pRows[i]->numOfRows == 0) { + (*index) += 1; pGroupResInfo->rowId = 0; continue; } - int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId; + int32_t numOfRowsToCopy = pRows[i]->numOfRows - pGroupResInfo->rowId; int32_t oldOffset = pGroupResInfo->rowId; /* @@ -4382,16 +4068,16 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_ pGroupResInfo->rowId += numOfRowsToCopy; } else { pGroupResInfo->rowId = 0; - pQInfo->groupIndex += 1; + (*index) += 1; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRows[i]->pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page); + char *in = getPosInResultPage(pRuntimeEnv, j, pRows[i], page); memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); } @@ -4422,10 +4108,9 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo, orderType); + int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo->pResult, pResultInfo->size, &pQInfo->groupIndex, orderType); pQuery->rec.rows += numOfResult; - assert(pQuery->rec.rows <= pQuery->rec.capacity); } @@ -4457,25 +4142,17 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo * pResultRowInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { + rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock); + blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); } if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - - // TODO refactor - if ((pTableQueryInfo->lastKey >= pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey <= pTableQueryInfo->win.ekey && (!ascQuery))) { - closeAllResultRows(pWindowResInfo); - pWindowResInfo->curIndex = pWindowResInfo->size - 1; - } else { - updateResultRowCurrentIndex(pWindowResInfo, pTableQueryInfo->lastKey, ascQuery); - } + updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); } } @@ -4511,7 +4188,7 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { } else { // there are results waiting for returned to client. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && - (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) && + (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) && (pRuntimeEnv->windowResInfo.size > 0)) { return true; } @@ -4547,7 +4224,9 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); + int32_t total = 0; STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL); + while(item) { STableIdInfo* pDst = (STableIdInfo*)data; pDst->uid = htobe64(item->uid); @@ -4555,9 +4234,14 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data pDst->key = htobe64(item->key); data += sizeof(STableIdInfo); + total++; + + qDebug("QInfo:%p set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo, item->tid, item->uid, item->key); item = taosHashIterate(pQInfo->arrTableIdInfo, item); } + qDebug("QInfo:%p set %d subscribe info", pQInfo, total); + // Check if query is completed or not for stable query or normal table query respectively. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (pQInfo->runtimeEnv.stableQuery) { @@ -4756,8 +4440,9 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w return key; } else { // do nothing - pQuery->window.skey = tw.skey; + pQuery->window.skey = tw.skey; pWindowResInfo->prevSKey = tw.skey; + pTableQueryInfo->lastKey = tw.skey; return tw.skey; } @@ -4767,24 +4452,12 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { SQuery *pQuery = pRuntimeEnv->pQuery; - - // get the first unclosed time window - bool assign = false; - for(int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - if (pRuntimeEnv->windowResInfo.pResult[i]->closed) { - continue; - } - - assign = true; - *start = pRuntimeEnv->windowResInfo.pResult[i]->win.skey; - } - - if (!assign) { - *start = pQuery->current->lastKey; + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert(*start <= pQuery->current->lastKey); + } else { + assert(*start >= pQuery->current->lastKey); } - assert(*start <= pQuery->current->lastKey); - // if queried with value filter, do NOT forward query start position if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { return true; @@ -4798,6 +4471,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4809,13 +4483,10 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { if (QUERY_IS_ASC_QUERY(pQuery)) { if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; pWindowResInfo->prevSKey = w.skey; } } else { getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; pWindowResInfo->prevSKey = w.skey; } @@ -4825,19 +4496,25 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { while (pQuery->limit.offset > 0) { STimeWindow tw = win; - if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; pWindowResInfo->prevSKey = win.skey; + + // current time window is aligned with blockInfo.window.ekey + // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; + if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { + pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + } } - // current window does not ended in current data block, try next data block - getNextTimeWindow(pQuery, &tw); if (pQuery->limit.offset == 0) { *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); return true; } + // current window does not ended in current data block, try next data block + getNextTimeWindow(pQuery, &tw); + /* * If the next time window still starts from current data block, * load the primary timestamp column first, and then find the start position for the next queried time window. @@ -4845,13 +4522,12 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required * time window resides in current data block. */ - if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); + if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { + + SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { + if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { pQuery->limit.offset -= 1; } @@ -4918,7 +4594,6 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) // update the query time window pQuery->window = cond.twindow; - if (pQInfo->tableGroupInfo.numOfTables == 0) { pQInfo->tableqinfoGroupInfo.numOfTables = 0; } else { @@ -4995,7 +4670,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->cur.vgroupIndex = -1; pRuntimeEnv->stableQuery = isSTableQuery; pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->groupbyNormalCol = isGroupbyNormalCol(pQuery->pGroupbyExpr); + pRuntimeEnv->groupbyColumn = isGroupbyNormalCol(pQuery->pGroupbyExpr); if (pTsBuf != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -5015,7 +4690,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; + if (pRuntimeEnv->groupbyColumn) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_INT; // group id @@ -5026,7 +4701,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } } - } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { + } else if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { int32_t numOfResultRows = getInitialPageNum(pQInfo); getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); @@ -5035,7 +4710,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyNormalCol) { + if (pRuntimeEnv->groupbyColumn) { type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); } else { type = TSDB_DATA_TYPE_TIMESTAMP; @@ -5141,7 +4816,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { pQuery->current = *pTableQueryInfo; doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); } @@ -5399,7 +5074,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } } - } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query + } else if (pRuntimeEnv->groupbyColumn) { // group-by on normal columns query while (pQInfo->groupIndex < numOfGroups) { SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); @@ -5438,10 +5113,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { scanMultiTableDataBlocks(pQInfo); pQInfo->groupIndex += 1; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + taosArrayDestroy(s); // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pWindowResInfo->size <= 0) { continue; } @@ -5468,8 +5143,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQInfo->groupIndex = currentGroupIndex; // restore the group index assert(pQuery->rec.rows == pWindowResInfo->size); - - clearClosedResultRows(pRuntimeEnv, &pRuntimeEnv->windowResInfo); + resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); break; } } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTsBuf == NULL && !isTSCompQuery(pQuery)) { @@ -5730,7 +5404,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } } -static void doSaveContext(SQInfo *pQInfo) { +static int32_t doSaveContext(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -5756,9 +5430,7 @@ static void doSaveContext(SQInfo *pQInfo) { pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + return (pRuntimeEnv->pSecQueryHandle == NULL)? -1:0; } static void doRestoreContext(SQInfo *pQInfo) { @@ -5766,6 +5438,7 @@ static void doRestoreContext(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + SWITCH_ORDER(pQuery->order.order); if (pRuntimeEnv->pTsBuf != NULL) { SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); @@ -5796,7 +5469,7 @@ static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { static void multiTableQueryProcess(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + SQuery *pQuery = pRuntimeEnv->pQuery; if (pQInfo->groupIndex > 0) { /* @@ -5805,9 +5478,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ if (QUERY_IS_INTERVAL_QUERY(pQuery)) { copyResToQueryResultBuf(pQInfo, pQuery); -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif } else { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); } @@ -5833,12 +5503,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { doCloseAllTimeWindowAfterScan(pQInfo); if (needReverseScan(pQuery)) { - doSaveContext(pQInfo); - - el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); - - doRestoreContext(pQInfo); + int32_t code = doSaveContext(pQInfo); + if (code == TSDB_CODE_SUCCESS) { + el = scanMultiTableDataBlocks(pQInfo); + qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); + doRestoreContext(pQInfo); + } else { + pQInfo->code = code; + } } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); } @@ -5852,12 +5524,15 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { } if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { - if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) { + int32_t code = mergeGroupResult(pQInfo); + if (code == TSDB_CODE_SUCCESS) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif + } else { // set the error code + pQInfo->code = code; } } else { // not a interval query copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); @@ -5867,7 +5542,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { qDebug("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *) param; SExprInfo* pExprInfo = (SExprInfo*) pSupport->exprList; @@ -5921,7 +5595,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { } } else { arithSup.pArithExpr = pExpr; - tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, + arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, getArithemicInputSrc); } } @@ -5944,7 +5618,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; * select count(*) from table_name group by status_column; */ -static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableAggregationProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -5969,7 +5643,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) limitResults(pRuntimeEnv); } -static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { +static void tableProjectionProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; @@ -6024,33 +5698,6 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) } } -static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - while (1) { - scanOneTableDataBlocks(pRuntimeEnv, start); - - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - finalizeQueryResult(pRuntimeEnv); - - // here we can ignore the records in case of no interpolation - // todo handle offset, in case of top/bottom interval query - if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) && pQuery->limit.offset > 0 && - pQuery->fillType == TSDB_FILL_NONE) { - // maxOutput <= 0, means current query does not generate any results - int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); - - int32_t c = (int32_t)(MIN(numOfClosed, pQuery->limit.offset)); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, c); - pQuery->limit.offset -= c; - } - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) { - break; - } - } -} - // handle time interval query on table static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); @@ -6058,11 +5705,10 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - int32_t numOfFilled = 0; - TSKEY newStartKey = TSKEY_INITIAL_VAL; + TSKEY newStartKey = QUERY_IS_ASC_QUERY(pQuery)? INT64_MIN:INT64_MAX; // skip blocks without load the actual data block from file if no filter condition present - if (!pRuntimeEnv->groupbyNormalCol) { + if (!pRuntimeEnv->groupbyColumn) { skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { setQueryStatus(pQuery, QUERY_COMPLETED); @@ -6070,59 +5716,40 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { } } - while (1) { - tableIntervalProcessImpl(pRuntimeEnv, newStartKey); + scanOneTableDataBlocks(pRuntimeEnv, newStartKey); + assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - pQInfo->groupIndex = 0; // always start from 0 - pQuery->rec.rows = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + finalizeQueryResult(pRuntimeEnv); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex); - } + // skip offset result rows + pQuery->rec.rows = 0; - // no result generated, abort - if (pQuery->rec.rows == 0 || pRuntimeEnv->groupbyNormalCol) { - break; + // not fill or no result generated during this query + if (pQuery->fillType == TSDB_FILL_NONE || pRuntimeEnv->windowResInfo.size == 0) { + // all data scanned, the group by normal column can return + int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); + if (pQuery->limit.offset > numOfClosed) { + return; } - doSecondaryArithmeticProcess(pQuery); - - // the offset is handled at prepare stage if no interpolation involved - if (pQuery->fillType == TSDB_FILL_NONE) { - limitResults(pRuntimeEnv); - break; - } else { - taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); - taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata); - numOfFilled = 0; + pQInfo->groupIndex = (int32_t) pQuery->limit.offset; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); - if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - limitResults(pRuntimeEnv); - break; - } + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + doSecondaryArithmeticProcess(pQuery); - // no result generated yet, continue retrieve data - pQuery->rec.rows = 0; - } - } + limitResults(pRuntimeEnv); + } else { - // all data scanned, the group by normal column can return - if (pRuntimeEnv->groupbyNormalCol) { // todo refactor with merge interval time result - // maxOutput <= 0, means current query does not generate any results - int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + doSecondaryArithmeticProcess(pQuery); - if ((pQuery->limit.offset > 0 && pQuery->limit.offset < numOfClosed) || pQuery->limit.offset == 0) { - // skip offset result rows - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (int32_t) pQuery->limit.offset); + taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); + taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage **)pQuery->sdata); - pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex); + int32_t numOfFilled = 0; + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); - doSecondaryArithmeticProcess(pQuery); + if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); } } @@ -6133,7 +5760,6 @@ static void tableQueryImpl(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; if (queryHasRemainResForTableQuery(pRuntimeEnv)) { - if (pQuery->fillType != TSDB_FILL_NONE) { /* * There are remain results that are not returned due to result interpolation @@ -6150,23 +5776,23 @@ static void tableQueryImpl(SQInfo *pQInfo) { return; } else { pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; // always start from 0 + assert(pRuntimeEnv->windowResInfo.size > 0); - if (pRuntimeEnv->windowResInfo.size > 0) { + if (pQInfo->groupIndex < pRuntimeEnv->windowResInfo.size) { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex); - - if (pQuery->rec.rows > 0) { - qDebug("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); + } - // there are not data remains - if (pRuntimeEnv->windowResInfo.size <= 0) { - qDebug("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total); - } + if (pQuery->rec.rows > 0) { + qDebug("QInfo:%p %" PRId64 " rows returned from group results, total:%" PRId64 "", pQInfo, pQuery->rec.rows, + pQuery->rec.total); + } - return; - } + // there are not data remains + if (pQuery->rec.rows <= 0 || pRuntimeEnv->windowResInfo.size <= pQInfo->groupIndex) { + qDebug("QInfo:%p query over, %" PRId64 " rows are returned", pQInfo, pQuery->rec.total); } + + return; } } @@ -6179,13 +5805,13 @@ static void tableQueryImpl(SQInfo *pQInfo) { STableQueryInfo* item = taosArrayGetP(g, 0); // group by normal column, sliding window query, interval query are handled by interval query processor - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) + if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); } else if (isFixedOutputQuery(pRuntimeEnv)) { - tableFixedOutputProcess(pQInfo, item); + tableAggregationProcess(pQInfo, item); } else { // diff/add/multiply/subtract/division assert(pQuery->checkBuffer == 1); - tableMultiOutputProcess(pQInfo, item); + tableProjectionProcess(pQInfo, item); } // record the total elapsed time @@ -6201,11 +5827,11 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyColumn))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || - isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); + pRuntimeEnv->groupbyColumn); sequentialTableProcess(pQInfo); } @@ -6241,13 +5867,13 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } } - assert(0); - return -1; + + return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value } bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { int32_t j = getColumnIndexInSource(pQueryMsg, pExprMsg, pTagCols); - return j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags; + return j != INT32_MIN; } static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { @@ -6274,12 +5900,14 @@ static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { return true; } -static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg) { +static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pExprMsg, SColumnInfo* pTagCols) { int32_t numOfTotal = pQueryMsg->numOfCols + pQueryMsg->numOfTags; if (pQueryMsg->numOfCols < 0 || pQueryMsg->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pQueryMsg, pQueryMsg->numOfCols, pQueryMsg->numOfTags); return false; - } else if (numOfTotal == 0) { + } + + if (numOfTotal == 0) { for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { SSqlFuncMsg* pFuncMsg = pExprMsg[i]; @@ -6293,6 +5921,12 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx } } + for(int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + if (!validateExprColumnInfo(pQueryMsg, pExprMsg[i], pTagCols)) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } + return true; } @@ -6324,7 +5958,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, SSqlFuncMsg ***pSecStageExpr, - char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { + char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols, char** sql) { int32_t code = TSDB_CODE_SUCCESS; if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -6356,7 +5990,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); pQueryMsg->tsOrder = htonl(pQueryMsg->tsOrder); pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); + pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); + pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); // query msg safety check if (!validateQueryMsg(pQueryMsg)) { @@ -6451,20 +6087,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; } - if (!validateQuerySourceCols(pQueryMsg, *pExpr)) { - code = TSDB_CODE_QRY_INVALID_MSG; - goto _cleanup; - } - if (pQueryMsg->secondStageOutput) { pExprMsg = (SSqlFuncMsg *)pMsg; *pSecStageExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); @@ -6498,10 +6125,6 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } - } else { -// if (!validateExprColumnInfo(pQueryMsg, pExprMsg)) { -// return TSDB_CODE_QRY_INVALID_MSG; -// } } pExprMsg = (SSqlFuncMsg *)pMsg; @@ -6518,13 +6141,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - (*groupbyCols)[i].colId = *(int16_t *)pMsg; + (*groupbyCols)[i].colId = htons(*(int16_t *)pMsg); pMsg += sizeof((*groupbyCols)[i].colId); - (*groupbyCols)[i].colIndex = *(int16_t *)pMsg; + (*groupbyCols)[i].colIndex = htons(*(int16_t *)pMsg); pMsg += sizeof((*groupbyCols)[i].colIndex); - (*groupbyCols)[i].flag = *(int16_t *)pMsg; + (*groupbyCols)[i].flag = htons(*(int16_t *)pMsg); pMsg += sizeof((*groupbyCols)[i].flag); memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name)); @@ -6580,17 +6203,22 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += pQueryMsg->tagCondLen; } - if (*pMsg != 0) { - size_t len = strlen(pMsg) + 1; - - *tbnameCond = malloc(len); + if (pQueryMsg->tbnameCondLen > 0) { + *tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); if (*tbnameCond == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } - strcpy(*tbnameCond, pMsg); - pMsg += len; + strncpy(*tbnameCond, pMsg, pQueryMsg->tbnameCondLen); + pMsg += pQueryMsg->tbnameCondLen; + } + + *sql = strndup(pMsg, pQueryMsg->sqlstrLen); + + if (!validateQuerySourceCols(pQueryMsg, *pExpr, *tagCols)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; } qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " @@ -6599,6 +6227,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); + qDebug("qmsg:%p, sql:%s", pQueryMsg, *sql); return TSDB_CODE_SUCCESS; _cleanup: @@ -6609,6 +6238,7 @@ _cleanup: tfree(*groupbyCols); tfree(*tagCols); tfree(*tagCond); + tfree(*sql); return code; } @@ -6681,7 +6311,15 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num } } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); + if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { + if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pQueryMsg->numOfTags) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } else { + if (j < PRIMARYKEY_TIMESTAMP_COL_INDEX || j >= pQueryMsg->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pQueryMsg->colList[j]; @@ -6705,6 +6343,7 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num if (pExprs[i].base.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].base.functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExprs[i].bytes; } + assert(isValidDataType(pExprs[i].type)); } @@ -6907,7 +6546,7 @@ static void calResultBufSize(SQuery* pQuery) { } static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery) { + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -7038,6 +6677,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; + pQInfo->sql = sql; pthread_mutex_init(&pQInfo->lock, NULL); tsem_init(&pQInfo->ready, 0, 0); @@ -7281,9 +6921,13 @@ static void freeQInfo(SQInfo *pQInfo) { doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); tfree(pQInfo->pBuf); + tfree(pQInfo->sql); + tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosHashCleanup(pQInfo->arrTableIdInfo); + taosArrayDestroy(pQInfo->groupResInfo.pRows); + pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -7377,6 +7021,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi int32_t code = TSDB_CODE_SUCCESS; + char *sql = NULL; char *tagCond = NULL; char *tbnameCond = NULL; SArray *pTableIdList = NULL; @@ -7389,7 +7034,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi SColumnInfo *pTagColumnInfo = NULL; SSqlGroupbyExpr *pGroupbyExpr = NULL; - code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo); + code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo, &sql); if (code != TSDB_CODE_SUCCESS) { goto _over; } @@ -7473,8 +7118,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi goto _over; } - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery); + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery, sql); + sql = NULL; pExprs = NULL; pSecExprs = NULL; pGroupbyExpr = NULL; @@ -7498,6 +7144,7 @@ _over: } free(pTagColumnInfo); + free(sql); free(pExprs); free(pSecExprs); @@ -7541,7 +7188,7 @@ static bool doBuildResCheck(SQInfo* pQInfo) { // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is // put into task to be executed. - assert(pQInfo->owner == taosGetPthreadId()); + assert(pQInfo->owner == taosGetSelfPthreadId()); pQInfo->owner = 0; pthread_mutex_unlock(&pQInfo->lock); @@ -7554,7 +7201,7 @@ static bool doBuildResCheck(SQInfo* pQInfo) { bool qTableQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; assert(pQInfo && pQInfo->signature == pQInfo); - int64_t threadId = taosGetPthreadId(); + int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { @@ -7703,7 +7350,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->completed = 1; // notify no more result to client } else { *continueExec = true; - qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); + qDebug("QInfo:%p has more results to retrieve", pQInfo); } return pQInfo->code; @@ -7981,9 +7628,9 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) { SQueryMgmt* pQueryMgmt = pQMgmt; qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); -// pthread_mutex_lock(&pQueryMgmt->lock); + pthread_mutex_lock(&pQueryMgmt->lock); pQueryMgmt->closed = true; -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); } @@ -8021,9 +7668,9 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { return NULL; } -// pthread_mutex_lock(&pQueryMgmt->lock); + pthread_mutex_lock(&pQueryMgmt->lock); if (pQueryMgmt->closed) { -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; return NULL; @@ -8031,7 +7678,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo; void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), (getMaximumIdleDurationSec()*1000)); -// pthread_mutex_unlock(&pQueryMgmt->lock); + pthread_mutex_unlock(&pQueryMgmt->lock); return handle; } @@ -8068,6 +7715,4 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); return 0; -} - - +} \ No newline at end of file diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 35e5906d1f797c2f53f3906bf93c6797ac766c3e..0d31948e59e520452605ac4577b695c53b3fad6c 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -158,8 +158,8 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } #if defined(USE_ARRAYLIST) - int32_t idx = vnodeHistobinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); - assert(idx >= 0 && idx <= (*pHisto)->maxEntries); + int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val); + assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL); if ((*pHisto)->elems[idx].val == val && idx >= 0) { (*pHisto)->elems[idx].num += 1; @@ -174,7 +174,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } assert((*pHisto)->elems[idx].val > val); - } else { + } else if ((*pHisto)->numOfElems > 0) { assert((*pHisto)->elems[(*pHisto)->numOfEntries].val < val); } @@ -356,7 +356,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { return 0; } -int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val) { +int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) { int32_t end = len - 1; int32_t start = 0; @@ -466,7 +466,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) { */ int64_t tHistogramSum(SHistogramInfo* pHisto, double v) { #if defined(USE_ARRAYLIST) - int32_t slotIdx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, v); + int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v); if (pHisto->elems[slotIdx].val != v) { slotIdx -= 1; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 5e5bc63675aa589093e38d4e7b733f263ae66efa..4b3a56e4ee503bfcd8a558bfe9050a787310650e 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -384,7 +384,12 @@ void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) pField->name[pName->n] = 0; pField->type = pType->type; - pField->bytes = pType->bytes; + if(pField->type < TSDB_DATA_TYPE_BOOL || pField->type > TSDB_DATA_TYPE_NCHAR){ + pField->bytes = 0; + } else { + pField->bytes = pType->bytes; + } + } void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type) { @@ -841,5 +846,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->keep = NULL; pDBInfo->update = -1; + pDBInfo->cachelast = 0; memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index bc7243830d836287cde808076cfb24de80fa574d..d45e76c2fd07f37afa0ffad0f9db65d5e1948c43 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -313,7 +313,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 // allocate buf if (availablePage == NULL) { - pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES); + pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES + 2); // add extract bytes in case of zipped buffer increased. } else { pi->pData = availablePage; } diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 98545c8ef3930587b6f7bd2c5b8d40ded0066d52..e243637333af5df05b02ee6e18c3a1387973a21f 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -238,6 +238,7 @@ static SKeyword keywordTable[] = { {"SUM_IRATE", TK_SUM_IRATE}, {"AVG_RATE", TK_AVG_RATE}, {"AVG_IRATE", TK_AVG_IRATE}, + {"CACHELAST", TK_CACHELAST}, }; static const char isIdChar[] = { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 65ac60e91fc9901e8bf96cca9b799b767b7e4d4a..dc01de0f92a0d591c67f3a156caaae34e318c962 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -20,18 +20,6 @@ #include "qExecutor.h" #include "qUtil.h" -static int32_t getResultRowKeyInfo(SResultRow* pResult, int16_t type, char** key, int16_t* bytes) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - *key = varDataVal(pResult->key); - *bytes = varDataLen(pResult->key); - } else { - *key = (char*) &pResult->win.skey; - *bytes = tDataTypeDesc[type].nSize; - } - - return 0; -} - int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; @@ -96,78 +84,9 @@ void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRo pResultRowInfo->curIndex = -1; pResultRowInfo->size = 0; - - pResultRowInfo->startTime = TSKEY_INITIAL_VAL; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } -void popFrontResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int32_t num) { - if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) { - return; - } - - int32_t numOfClosed = numOfClosedResultRows(pResultRowInfo); - assert(num >= 0 && num <= numOfClosed); - - int16_t type = pResultRowInfo->type; - int64_t uid = getResultInfoUId(pRuntimeEnv); - - char *key = NULL; - int16_t bytes = -1; - - for (int32_t i = 0; i < num; ++i) { - SResultRow *pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { // remove the window slot from hash table - getResultRowKeyInfo(pResult, type, &key, &bytes); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - } else { - break; - } - } - - int32_t remain = pResultRowInfo->size - num; - - // clear all the closed windows from the window list - for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type); - } - - // move the unclosed window in the front of the window list - for (int32_t k = remain; k < pResultRowInfo->size; ++k) { - SResultRow *pWindowRes = pResultRowInfo->pResult[k]; - clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); - } - - pResultRowInfo->size = remain; - - for (int32_t k = 0; k < pResultRowInfo->size; ++k) { - SResultRow *pResult = pResultRowInfo->pResult[k]; - getResultRowKeyInfo(pResult, type, &key, &bytes); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - - int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - assert(p != NULL); - - int32_t v = (*p - num); - assert(v >= 0 && v <= pResultRowInfo->size); - - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); - } - - pResultRowInfo->curIndex = -1; -} - -void clearClosedResultRows(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { - if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) { - return; - } - - int32_t numOfClosed = numOfClosedResultRows(pResultRowInfo); - popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, numOfClosed); -} - int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { int32_t i = 0; while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { @@ -181,45 +100,12 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) { assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - if (pResultRowInfo->pResult[i]->closed) { + SResultRow* pRow = pResultRowInfo->pResult[i]; + if (pRow->closed) { continue; } - pResultRowInfo->pResult[i]->closed = true; - } -} - -/* - * remove the results that are not the FIRST time window that spreads beyond the - * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time. - * NOTE: remove redundant, only when the result set order equals to traverse order - */ -void removeRedundantResultRows(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) { - assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); - if (pResultRowInfo->size <= 1) { - return; - } - - // get the result order - int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1; - if (order != resultOrder) { - return; - } - - int32_t i = 0; - if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey; - while (i < pResultRowInfo->size && (ekey < lastKey)) { - ++i; - } - } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) { - ++i; - } - } - - if (i < pResultRowInfo->size) { - pResultRowInfo->size = (i + 1); + pRow->closed = true; } } @@ -263,47 +149,6 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 } } -/** - * The source window result pos attribution of the source window result does not assign to the destination, - * since the attribute of "Pos" is bound to each window result when the window result is created in the - * disk-based result buffer. - */ -void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src, int16_t type) { - dst->numOfRows = src->numOfRows; - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - dst->key = realloc(dst->key, varDataTLen(src->key)); - varDataCopy(dst->key, src->key); - } else { - dst->win = src->win; - } - dst->closed = src->closed; - - int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; - - for (int32_t i = 0; i < nOutputCols; ++i) { - SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i); - SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i); - -// char *buf = pDst->interResultBuf; - memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes); -// pDst->interResultBuf = buf; // restore the allocated buffer - - // copy the result info struct -// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes); - - // copy the output buffer data from src to dst, the position info keep unchanged - tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId); - char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); - - tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId); - char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage); - size_t s = pRuntimeEnv->pQuery->pExpr1[i].bytes; - - memcpy(dstBuf, srcBuf, s); - } -} - SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) { assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput); return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); @@ -383,18 +228,4 @@ void* destroyResultRowPool(SResultRowPool* p) { tfree(p); return NULL; -} - -uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { - if (!pRuntimeEnv->stableQuery) { - return 0; // for simple table query, the uid is always set to be 0; - } - - SQuery* pQuery = pRuntimeEnv->pQuery; - if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) { - return 0; - } - - STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); - return id->uid; } \ No newline at end of file diff --git a/src/query/src/sql.c b/src/query/src/sql.c index fe82db72a6b4696dfdbcf15c48919fe6b4194a61..7c10a4ce6b568940e05dfb82763f4921aea78a8d 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -97,27 +97,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 276 +#define YYNOCODE 278 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - int yy42; - SQuerySQL* yy84; - SCreatedTableInfo yy96; - SArray* yy131; - SCreateDBInfo yy148; - TAOS_FIELD yy163; - SLimitVal yy284; - SCreateAcctSQL yy309; - tSQLExpr* yy420; - int64_t yy459; - tSQLExprList* yy478; - SSubclauseInfo* yy513; - tVariant yy516; - SIntervalVal yy530; - SCreateTableSQL* yy538; + SQuerySQL* yy4; + SSubclauseInfo* yy13; + int yy70; + SCreatedTableInfo yy84; + SIntervalVal yy222; + TAOS_FIELD yy363; + tSQLExprList* yy382; + int64_t yy387; + SArray* yy403; + SLimitVal yy404; + SCreateTableSQL* yy436; + SCreateAcctSQL yy463; + SCreateDBInfo yy478; + tVariant yy488; + tSQLExpr* yy522; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -127,17 +127,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 257 -#define YYNRULE 236 -#define YYNTOKEN 207 -#define YY_MAX_SHIFT 256 -#define YY_MIN_SHIFTREDUCE 426 -#define YY_MAX_SHIFTREDUCE 661 -#define YY_ERROR_ACTION 662 -#define YY_ACCEPT_ACTION 663 -#define YY_NO_ACTION 664 -#define YY_MIN_REDUCE 665 -#define YY_MAX_REDUCE 900 +#define YYNSTATE 258 +#define YYNRULE 239 +#define YYNTOKEN 208 +#define YY_MAX_SHIFT 257 +#define YY_MIN_SHIFTREDUCE 430 +#define YY_MAX_SHIFTREDUCE 668 +#define YY_ERROR_ACTION 669 +#define YY_ACCEPT_ACTION 670 +#define YY_NO_ACTION 671 +#define YY_MIN_REDUCE 672 +#define YY_MAX_REDUCE 910 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -203,226 +203,228 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (579) +#define YY_ACTTAB_COUNT (585) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 143, 469, 663, 256, 469, 162, 254, 12, 814, 470, - /* 10 */ 887, 142, 470, 37, 38, 147, 39, 40, 803, 233, - /* 20 */ 173, 31, 884, 469, 209, 43, 41, 45, 42, 64, - /* 30 */ 883, 470, 163, 36, 35, 105, 143, 34, 33, 32, - /* 40 */ 37, 38, 803, 39, 40, 168, 888, 173, 31, 110, - /* 50 */ 790, 209, 43, 41, 45, 42, 194, 780, 22, 782, - /* 60 */ 36, 35, 811, 882, 34, 33, 32, 427, 428, 429, - /* 70 */ 430, 431, 432, 433, 434, 435, 436, 437, 438, 255, - /* 80 */ 37, 38, 184, 39, 40, 538, 224, 173, 31, 143, - /* 90 */ 197, 209, 43, 41, 45, 42, 165, 23, 167, 888, - /* 100 */ 36, 35, 242, 57, 34, 33, 32, 179, 841, 38, - /* 110 */ 204, 39, 40, 231, 230, 173, 31, 49, 792, 209, - /* 120 */ 43, 41, 45, 42, 253, 252, 98, 615, 36, 35, - /* 130 */ 178, 781, 34, 33, 32, 788, 50, 17, 222, 249, - /* 140 */ 248, 221, 220, 219, 247, 218, 246, 245, 244, 217, - /* 150 */ 243, 764, 792, 752, 753, 754, 755, 756, 757, 758, - /* 160 */ 759, 760, 761, 762, 763, 765, 39, 40, 110, 180, - /* 170 */ 173, 31, 228, 227, 209, 43, 41, 45, 42, 34, - /* 180 */ 33, 32, 9, 36, 35, 65, 120, 34, 33, 32, - /* 190 */ 172, 628, 18, 13, 619, 110, 622, 210, 625, 28, - /* 200 */ 172, 628, 110, 159, 619, 187, 622, 224, 625, 155, - /* 210 */ 172, 628, 191, 190, 619, 156, 622, 66, 625, 92, - /* 220 */ 91, 150, 169, 170, 36, 35, 208, 840, 34, 33, - /* 230 */ 32, 160, 169, 170, 617, 250, 573, 43, 41, 45, - /* 240 */ 42, 706, 169, 170, 133, 36, 35, 783, 18, 34, - /* 250 */ 33, 32, 206, 104, 61, 28, 17, 792, 249, 248, - /* 260 */ 28, 62, 145, 247, 23, 246, 245, 244, 146, 243, - /* 270 */ 618, 715, 78, 82, 133, 193, 565, 23, 87, 90, - /* 280 */ 81, 769, 158, 196, 767, 768, 84, 631, 44, 770, - /* 290 */ 23, 772, 773, 771, 148, 774, 80, 149, 44, 627, - /* 300 */ 176, 242, 789, 707, 3, 124, 133, 23, 44, 627, - /* 310 */ 72, 68, 71, 177, 626, 789, 596, 597, 570, 627, - /* 320 */ 805, 63, 557, 19, 626, 554, 229, 555, 789, 556, - /* 330 */ 171, 137, 135, 29, 626, 153, 583, 95, 94, 93, - /* 340 */ 107, 154, 587, 234, 588, 789, 48, 647, 15, 52, - /* 350 */ 152, 14, 629, 181, 182, 141, 14, 621, 620, 624, - /* 360 */ 623, 546, 89, 88, 212, 24, 53, 547, 24, 151, - /* 370 */ 4, 48, 561, 144, 562, 77, 76, 11, 10, 897, - /* 380 */ 559, 851, 560, 103, 101, 791, 850, 174, 847, 846, - /* 390 */ 175, 232, 813, 833, 818, 820, 106, 832, 121, 122, - /* 400 */ 28, 123, 102, 119, 717, 216, 139, 26, 225, 714, - /* 410 */ 195, 582, 226, 896, 74, 895, 893, 125, 735, 27, - /* 420 */ 198, 25, 164, 202, 140, 558, 704, 83, 702, 85, - /* 430 */ 86, 700, 699, 183, 54, 134, 697, 696, 695, 694, - /* 440 */ 693, 136, 691, 689, 46, 687, 685, 802, 683, 138, - /* 450 */ 51, 58, 59, 834, 207, 205, 199, 203, 201, 30, - /* 460 */ 79, 235, 236, 237, 238, 239, 240, 241, 251, 161, - /* 470 */ 661, 214, 215, 186, 185, 660, 189, 157, 188, 69, - /* 480 */ 659, 652, 192, 60, 196, 166, 567, 698, 56, 128, - /* 490 */ 96, 692, 736, 126, 130, 97, 127, 129, 131, 132, - /* 500 */ 684, 1, 584, 787, 2, 108, 200, 117, 113, 111, - /* 510 */ 112, 114, 115, 116, 589, 118, 109, 5, 6, 20, - /* 520 */ 21, 630, 8, 7, 632, 211, 16, 213, 67, 510, - /* 530 */ 65, 506, 504, 503, 502, 499, 473, 223, 24, 70, - /* 540 */ 47, 73, 540, 539, 537, 55, 494, 492, 484, 490, - /* 550 */ 75, 486, 488, 482, 480, 511, 509, 508, 507, 505, - /* 560 */ 501, 500, 48, 471, 442, 440, 99, 665, 664, 664, - /* 570 */ 664, 664, 664, 664, 664, 664, 664, 664, 100, + /* 0 */ 143, 473, 143, 23, 670, 257, 165, 545, 824, 474, + /* 10 */ 897, 168, 898, 37, 38, 12, 39, 40, 813, 23, + /* 20 */ 173, 31, 473, 473, 209, 43, 41, 45, 42, 802, + /* 30 */ 474, 474, 163, 36, 35, 231, 230, 34, 33, 32, + /* 40 */ 37, 38, 798, 39, 40, 813, 105, 173, 31, 162, + /* 50 */ 255, 209, 43, 41, 45, 42, 176, 178, 799, 194, + /* 60 */ 36, 35, 233, 821, 34, 33, 32, 431, 432, 433, + /* 70 */ 434, 435, 436, 437, 438, 439, 440, 441, 442, 256, + /* 80 */ 802, 143, 184, 63, 179, 37, 38, 224, 39, 40, + /* 90 */ 167, 898, 173, 31, 800, 29, 209, 43, 41, 45, + /* 100 */ 42, 110, 197, 791, 57, 36, 35, 251, 210, 34, + /* 110 */ 33, 32, 110, 17, 222, 250, 249, 221, 220, 219, + /* 120 */ 248, 218, 247, 246, 245, 217, 244, 243, 622, 772, + /* 130 */ 802, 760, 761, 762, 763, 764, 765, 766, 767, 768, + /* 140 */ 769, 770, 771, 773, 774, 242, 38, 180, 39, 40, + /* 150 */ 228, 227, 173, 31, 110, 18, 209, 43, 41, 45, + /* 160 */ 42, 851, 28, 204, 110, 36, 35, 23, 187, 34, + /* 170 */ 33, 32, 850, 39, 40, 191, 190, 173, 31, 224, + /* 180 */ 624, 209, 43, 41, 45, 42, 34, 33, 32, 9, + /* 190 */ 36, 35, 65, 120, 34, 33, 32, 172, 635, 638, + /* 200 */ 66, 626, 104, 629, 177, 632, 799, 172, 635, 28, + /* 210 */ 13, 626, 206, 629, 61, 632, 625, 172, 635, 142, + /* 220 */ 572, 626, 23, 629, 62, 632, 155, 196, 147, 169, + /* 230 */ 170, 793, 156, 208, 603, 604, 92, 91, 150, 169, + /* 240 */ 170, 894, 713, 580, 17, 133, 250, 249, 893, 169, + /* 250 */ 170, 248, 64, 247, 246, 245, 892, 244, 243, 229, + /* 260 */ 778, 799, 80, 776, 777, 590, 18, 242, 779, 107, + /* 270 */ 781, 782, 780, 28, 783, 784, 43, 41, 45, 42, + /* 280 */ 159, 790, 22, 792, 36, 35, 160, 171, 34, 33, + /* 290 */ 32, 722, 564, 193, 133, 561, 44, 562, 52, 563, + /* 300 */ 158, 254, 253, 98, 36, 35, 44, 634, 34, 33, + /* 310 */ 32, 23, 145, 3, 124, 53, 44, 634, 146, 72, + /* 320 */ 68, 71, 633, 181, 182, 148, 714, 634, 4, 133, + /* 330 */ 78, 82, 633, 137, 135, 149, 87, 90, 81, 95, + /* 340 */ 94, 93, 633, 153, 84, 594, 577, 154, 234, 48, + /* 350 */ 799, 19, 49, 152, 595, 654, 636, 141, 15, 14, + /* 360 */ 14, 628, 627, 631, 630, 553, 212, 151, 554, 24, + /* 370 */ 24, 50, 48, 77, 76, 11, 10, 568, 566, 569, + /* 380 */ 567, 89, 88, 103, 101, 907, 144, 801, 861, 860, + /* 390 */ 174, 857, 856, 175, 823, 232, 565, 828, 830, 106, + /* 400 */ 843, 815, 842, 121, 122, 28, 119, 123, 195, 724, + /* 410 */ 216, 139, 26, 225, 721, 226, 906, 74, 102, 905, + /* 420 */ 903, 125, 742, 27, 25, 140, 711, 83, 589, 709, + /* 430 */ 85, 86, 707, 706, 183, 134, 704, 703, 702, 701, + /* 440 */ 198, 700, 136, 698, 696, 694, 692, 690, 138, 164, + /* 450 */ 58, 54, 59, 202, 51, 844, 46, 812, 207, 205, + /* 460 */ 203, 201, 199, 30, 79, 235, 236, 237, 238, 239, + /* 470 */ 240, 161, 214, 241, 252, 215, 668, 186, 185, 157, + /* 480 */ 69, 667, 188, 189, 666, 659, 192, 196, 166, 574, + /* 490 */ 705, 591, 56, 96, 132, 743, 126, 128, 127, 129, + /* 500 */ 130, 699, 111, 131, 1, 97, 116, 112, 113, 114, + /* 510 */ 691, 797, 60, 117, 115, 118, 2, 20, 108, 200, + /* 520 */ 6, 596, 109, 5, 7, 637, 21, 8, 211, 16, + /* 530 */ 213, 639, 67, 65, 514, 510, 508, 507, 506, 503, + /* 540 */ 477, 223, 70, 47, 73, 75, 24, 547, 546, 544, + /* 550 */ 55, 498, 496, 488, 494, 490, 492, 486, 484, 516, + /* 560 */ 515, 513, 512, 511, 509, 505, 504, 48, 475, 446, + /* 570 */ 444, 672, 671, 671, 671, 671, 671, 671, 671, 671, + /* 580 */ 671, 671, 671, 99, 100, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 264, 1, 208, 209, 1, 210, 211, 264, 211, 9, - /* 10 */ 274, 264, 9, 13, 14, 264, 16, 17, 248, 211, - /* 20 */ 20, 21, 264, 1, 24, 25, 26, 27, 28, 216, - /* 30 */ 264, 9, 262, 33, 34, 211, 264, 37, 38, 39, - /* 40 */ 13, 14, 248, 16, 17, 273, 274, 20, 21, 211, - /* 50 */ 242, 24, 25, 26, 27, 28, 262, 244, 245, 246, - /* 60 */ 33, 34, 265, 264, 37, 38, 39, 45, 46, 47, + /* 0 */ 266, 1, 266, 212, 209, 210, 229, 5, 212, 9, + /* 10 */ 276, 275, 276, 13, 14, 266, 16, 17, 250, 212, + /* 20 */ 20, 21, 1, 1, 24, 25, 26, 27, 28, 252, + /* 30 */ 9, 9, 264, 33, 34, 33, 34, 37, 38, 39, + /* 40 */ 13, 14, 251, 16, 17, 250, 212, 20, 21, 211, + /* 50 */ 212, 24, 25, 26, 27, 28, 249, 229, 251, 264, + /* 60 */ 33, 34, 212, 267, 37, 38, 39, 45, 46, 47, /* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 80 */ 13, 14, 60, 16, 17, 5, 76, 20, 21, 264, - /* 90 */ 266, 24, 25, 26, 27, 28, 228, 211, 273, 274, - /* 100 */ 33, 34, 78, 103, 37, 38, 39, 66, 270, 14, - /* 110 */ 272, 16, 17, 33, 34, 20, 21, 104, 250, 24, - /* 120 */ 25, 26, 27, 28, 63, 64, 65, 100, 33, 34, - /* 130 */ 228, 0, 37, 38, 39, 249, 123, 85, 86, 87, - /* 140 */ 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - /* 150 */ 98, 227, 250, 229, 230, 231, 232, 233, 234, 235, - /* 160 */ 236, 237, 238, 239, 240, 241, 16, 17, 211, 128, - /* 170 */ 20, 21, 131, 132, 24, 25, 26, 27, 28, 37, - /* 180 */ 38, 39, 99, 33, 34, 102, 103, 37, 38, 39, - /* 190 */ 1, 2, 99, 44, 5, 211, 7, 15, 9, 106, - /* 200 */ 1, 2, 211, 264, 5, 127, 7, 76, 9, 60, - /* 210 */ 1, 2, 134, 135, 5, 66, 7, 216, 9, 70, - /* 220 */ 71, 72, 33, 34, 33, 34, 37, 270, 37, 38, - /* 230 */ 39, 264, 33, 34, 1, 228, 37, 25, 26, 27, - /* 240 */ 28, 215, 33, 34, 218, 33, 34, 246, 99, 37, - /* 250 */ 38, 39, 268, 99, 270, 106, 85, 250, 87, 88, - /* 260 */ 106, 270, 264, 92, 211, 94, 95, 96, 264, 98, - /* 270 */ 37, 215, 61, 62, 218, 126, 100, 211, 67, 68, - /* 280 */ 69, 227, 133, 107, 230, 231, 75, 105, 99, 235, - /* 290 */ 211, 237, 238, 239, 264, 241, 73, 264, 99, 110, - /* 300 */ 247, 78, 249, 215, 61, 62, 218, 211, 99, 110, - /* 310 */ 67, 68, 69, 247, 125, 249, 116, 117, 104, 110, - /* 320 */ 248, 251, 2, 109, 125, 5, 247, 7, 249, 9, - /* 330 */ 59, 61, 62, 263, 125, 264, 100, 67, 68, 69, - /* 340 */ 104, 264, 100, 247, 100, 249, 104, 100, 104, 104, - /* 350 */ 264, 104, 100, 33, 34, 264, 104, 5, 5, 7, - /* 360 */ 7, 100, 73, 74, 100, 104, 121, 100, 104, 264, - /* 370 */ 99, 104, 5, 264, 7, 129, 130, 129, 130, 250, - /* 380 */ 5, 243, 7, 61, 62, 250, 243, 243, 243, 243, - /* 390 */ 243, 243, 211, 271, 211, 211, 211, 271, 211, 211, - /* 400 */ 106, 211, 59, 252, 211, 211, 211, 211, 211, 211, - /* 410 */ 248, 110, 211, 211, 211, 211, 211, 211, 211, 211, - /* 420 */ 267, 211, 267, 267, 211, 105, 211, 211, 211, 211, - /* 430 */ 211, 211, 211, 211, 120, 211, 211, 211, 211, 211, - /* 440 */ 211, 211, 211, 211, 119, 211, 211, 261, 211, 211, - /* 450 */ 122, 212, 212, 212, 114, 118, 111, 113, 112, 124, - /* 460 */ 84, 83, 49, 80, 82, 53, 81, 79, 76, 212, - /* 470 */ 5, 212, 212, 5, 136, 5, 5, 212, 136, 216, - /* 480 */ 5, 86, 127, 104, 107, 1, 100, 212, 108, 220, - /* 490 */ 213, 212, 226, 225, 221, 213, 224, 223, 222, 219, - /* 500 */ 212, 217, 100, 248, 214, 99, 99, 254, 258, 260, - /* 510 */ 259, 257, 256, 255, 100, 253, 99, 99, 115, 104, - /* 520 */ 104, 100, 99, 115, 105, 101, 99, 101, 73, 9, - /* 530 */ 102, 5, 5, 5, 5, 5, 77, 15, 104, 73, - /* 540 */ 16, 130, 5, 5, 100, 99, 5, 5, 5, 5, - /* 550 */ 130, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 560 */ 5, 5, 104, 77, 59, 58, 21, 0, 275, 275, - /* 570 */ 275, 275, 275, 275, 275, 275, 275, 275, 21, 275, - /* 580 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 590 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 600 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 610 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 620 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 630 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 640 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 650 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 660 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 670 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 680 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 690 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 700 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 710 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 720 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 730 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 740 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 750 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 760 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 770 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, - /* 780 */ 275, 275, 275, 275, 275, 275, + /* 80 */ 252, 266, 60, 253, 66, 13, 14, 76, 16, 17, + /* 90 */ 275, 276, 20, 21, 244, 265, 24, 25, 26, 27, + /* 100 */ 28, 212, 268, 0, 104, 33, 34, 229, 15, 37, + /* 110 */ 38, 39, 212, 85, 86, 87, 88, 89, 90, 91, + /* 120 */ 92, 93, 94, 95, 96, 97, 98, 99, 101, 228, + /* 130 */ 252, 230, 231, 232, 233, 234, 235, 236, 237, 238, + /* 140 */ 239, 240, 241, 242, 243, 78, 14, 129, 16, 17, + /* 150 */ 132, 133, 20, 21, 212, 100, 24, 25, 26, 27, + /* 160 */ 28, 272, 107, 274, 212, 33, 34, 212, 128, 37, + /* 170 */ 38, 39, 272, 16, 17, 135, 136, 20, 21, 76, + /* 180 */ 1, 24, 25, 26, 27, 28, 37, 38, 39, 100, + /* 190 */ 33, 34, 103, 104, 37, 38, 39, 1, 2, 106, + /* 200 */ 217, 5, 100, 7, 249, 9, 251, 1, 2, 107, + /* 210 */ 44, 5, 270, 7, 272, 9, 37, 1, 2, 266, + /* 220 */ 101, 5, 212, 7, 272, 9, 60, 108, 266, 33, + /* 230 */ 34, 248, 66, 37, 117, 118, 70, 71, 72, 33, + /* 240 */ 34, 266, 216, 37, 85, 219, 87, 88, 266, 33, + /* 250 */ 34, 92, 217, 94, 95, 96, 266, 98, 99, 249, + /* 260 */ 228, 251, 73, 231, 232, 101, 100, 78, 236, 105, + /* 270 */ 238, 239, 240, 107, 242, 243, 25, 26, 27, 28, + /* 280 */ 266, 246, 247, 248, 33, 34, 266, 59, 37, 38, + /* 290 */ 39, 216, 2, 127, 219, 5, 100, 7, 105, 9, + /* 300 */ 134, 63, 64, 65, 33, 34, 100, 111, 37, 38, + /* 310 */ 39, 212, 266, 61, 62, 122, 100, 111, 266, 67, + /* 320 */ 68, 69, 126, 33, 34, 266, 216, 111, 100, 219, + /* 330 */ 61, 62, 126, 61, 62, 266, 67, 68, 69, 67, + /* 340 */ 68, 69, 126, 266, 75, 101, 105, 266, 249, 105, + /* 350 */ 251, 110, 105, 266, 101, 101, 101, 266, 105, 105, + /* 360 */ 105, 5, 5, 7, 7, 101, 101, 266, 101, 105, + /* 370 */ 105, 124, 105, 130, 131, 130, 131, 5, 5, 7, + /* 380 */ 7, 73, 74, 61, 62, 252, 266, 252, 245, 245, + /* 390 */ 245, 245, 245, 245, 212, 245, 106, 212, 212, 212, + /* 400 */ 273, 250, 273, 212, 212, 107, 254, 212, 250, 212, + /* 410 */ 212, 212, 212, 212, 212, 212, 212, 212, 59, 212, + /* 420 */ 212, 212, 212, 212, 212, 212, 212, 212, 111, 212, + /* 430 */ 212, 212, 212, 212, 212, 212, 212, 212, 212, 212, + /* 440 */ 269, 212, 212, 212, 212, 212, 212, 212, 212, 269, + /* 450 */ 213, 121, 213, 269, 123, 213, 120, 263, 115, 119, + /* 460 */ 114, 113, 112, 125, 84, 83, 49, 80, 82, 53, + /* 470 */ 81, 213, 213, 79, 76, 213, 5, 5, 137, 213, + /* 480 */ 217, 5, 137, 5, 5, 86, 128, 108, 1, 101, + /* 490 */ 213, 101, 109, 214, 220, 227, 226, 221, 225, 224, + /* 500 */ 222, 213, 262, 223, 218, 214, 257, 261, 260, 259, + /* 510 */ 213, 250, 105, 256, 258, 255, 215, 105, 100, 100, + /* 520 */ 116, 101, 100, 100, 116, 101, 105, 100, 102, 100, + /* 530 */ 102, 106, 73, 103, 9, 5, 5, 5, 5, 5, + /* 540 */ 77, 15, 73, 16, 131, 131, 105, 5, 5, 101, + /* 550 */ 100, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 560 */ 5, 5, 5, 5, 5, 5, 5, 105, 77, 59, + /* 570 */ 58, 0, 277, 277, 277, 277, 277, 277, 277, 277, + /* 580 */ 277, 277, 277, 21, 21, 277, 277, 277, 277, 277, + /* 590 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 600 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 610 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 620 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 630 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 640 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 650 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 660 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 670 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 680 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 690 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 700 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 710 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 720 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 730 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 740 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 750 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 760 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 770 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 780 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 790 */ 277, 277, 277, }; -#define YY_SHIFT_COUNT (256) +#define YY_SHIFT_COUNT (257) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (567) +#define YY_SHIFT_MAX (571) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 149, 52, 171, 10, 189, 209, 3, 3, 3, 3, - /* 10 */ 3, 3, 0, 22, 209, 320, 320, 320, 93, 3, - /* 20 */ 3, 3, 131, 3, 3, 223, 24, 24, 579, 199, - /* 30 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, - /* 40 */ 209, 209, 209, 209, 209, 209, 209, 320, 320, 80, - /* 50 */ 80, 80, 80, 80, 80, 80, 154, 3, 3, 3, - /* 60 */ 3, 200, 200, 214, 3, 3, 3, 3, 3, 3, - /* 70 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - /* 80 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - /* 90 */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - /* 100 */ 3, 3, 3, 3, 294, 343, 343, 301, 301, 301, - /* 110 */ 343, 314, 328, 325, 340, 337, 344, 346, 345, 335, - /* 120 */ 294, 343, 343, 343, 10, 343, 376, 378, 413, 383, - /* 130 */ 382, 412, 385, 388, 343, 392, 343, 392, 343, 579, - /* 140 */ 579, 27, 67, 67, 67, 95, 150, 212, 212, 212, - /* 150 */ 211, 191, 191, 191, 191, 243, 270, 41, 78, 142, - /* 160 */ 142, 83, 61, 176, 236, 242, 244, 247, 252, 352, - /* 170 */ 353, 233, 271, 182, 13, 245, 261, 264, 267, 246, - /* 180 */ 248, 367, 375, 289, 322, 465, 338, 468, 470, 342, - /* 190 */ 471, 475, 395, 355, 377, 386, 380, 379, 402, 406, - /* 200 */ 484, 407, 414, 417, 415, 403, 416, 408, 421, 418, - /* 210 */ 419, 423, 424, 427, 426, 428, 455, 520, 526, 527, - /* 220 */ 528, 529, 530, 459, 522, 466, 524, 411, 420, 434, - /* 230 */ 537, 538, 444, 446, 434, 541, 542, 543, 544, 546, - /* 240 */ 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, - /* 250 */ 458, 486, 545, 557, 505, 507, 567, + /* 0 */ 166, 28, 159, 11, 196, 216, 21, 21, 21, 21, + /* 10 */ 21, 21, 0, 22, 216, 290, 290, 290, 55, 21, + /* 20 */ 21, 21, 103, 21, 21, 189, 67, 67, 585, 206, + /* 30 */ 216, 216, 216, 216, 216, 216, 216, 216, 216, 216, + /* 40 */ 216, 216, 216, 216, 216, 216, 216, 290, 290, 2, + /* 50 */ 2, 2, 2, 2, 2, 2, 102, 21, 21, 21, + /* 60 */ 21, 117, 117, 241, 21, 21, 21, 21, 21, 21, + /* 70 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + /* 80 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + /* 90 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + /* 100 */ 21, 21, 21, 21, 298, 359, 359, 317, 317, 317, + /* 110 */ 359, 330, 331, 336, 343, 340, 346, 348, 350, 338, + /* 120 */ 298, 359, 359, 359, 11, 359, 380, 382, 417, 387, + /* 130 */ 386, 416, 389, 394, 359, 398, 359, 398, 359, 585, + /* 140 */ 585, 27, 72, 72, 72, 132, 157, 251, 251, 251, + /* 150 */ 269, 271, 271, 271, 271, 252, 272, 18, 40, 149, + /* 160 */ 149, 89, 238, 119, 164, 244, 253, 254, 255, 356, + /* 170 */ 357, 179, 228, 93, 247, 193, 264, 265, 267, 243, + /* 180 */ 245, 372, 373, 308, 322, 471, 341, 472, 476, 345, + /* 190 */ 478, 479, 399, 358, 379, 388, 383, 407, 390, 418, + /* 200 */ 487, 419, 420, 422, 412, 404, 421, 408, 424, 423, + /* 210 */ 425, 427, 426, 429, 428, 430, 459, 525, 530, 531, + /* 220 */ 532, 533, 534, 463, 526, 469, 527, 413, 414, 441, + /* 230 */ 542, 543, 448, 450, 441, 546, 547, 548, 549, 550, + /* 240 */ 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, + /* 250 */ 561, 462, 491, 562, 563, 510, 512, 571, }; #define YY_REDUCE_COUNT (140) -#define YY_REDUCE_MIN (-264) -#define YY_REDUCE_MAX (290) +#define YY_REDUCE_MIN (-266) +#define YY_REDUCE_MAX (301) static const short yy_reduce_ofst[] = { - /* 0 */ -206, -76, 54, -187, -228, -175, -162, -16, 53, 66, - /* 10 */ 79, 96, -203, -205, -264, -132, -98, 7, -230, -176, - /* 20 */ -43, -9, 1, -192, -114, 26, 56, 88, 70, -257, - /* 30 */ -253, -249, -242, -234, -201, -61, -33, -2, 4, 30, - /* 40 */ 33, 71, 77, 86, 91, 105, 109, 129, 135, 138, - /* 50 */ 143, 144, 145, 146, 147, 148, 72, 181, 183, 184, - /* 60 */ 185, 122, 126, 151, 187, 188, 190, 193, 194, 195, - /* 70 */ 196, 197, 198, 201, 202, 203, 204, 205, 206, 207, - /* 80 */ 208, 210, 213, 215, 216, 217, 218, 219, 220, 221, - /* 90 */ 222, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 100 */ 234, 235, 237, 238, 162, 239, 240, 153, 155, 156, - /* 110 */ 241, 186, 249, 251, 250, 254, 256, 258, 253, 262, - /* 120 */ 255, 257, 259, 260, 263, 265, 266, 268, 272, 269, - /* 130 */ 274, 273, 276, 280, 275, 277, 279, 282, 288, 284, - /* 140 */ 290, + /* 0 */ -205, -99, 32, 35, -264, -185, -111, -58, -193, -45, + /* 10 */ 10, 99, -204, -162, -266, -223, -172, -122, -232, -166, + /* 20 */ -100, -48, -17, -150, -209, 26, 75, 110, -170, -251, + /* 30 */ -47, -38, -25, -18, -10, 14, 20, 46, 52, 59, + /* 40 */ 69, 77, 81, 87, 91, 101, 120, 133, 135, 143, + /* 50 */ 144, 145, 146, 147, 148, 150, 151, 182, 185, 186, + /* 60 */ 187, 127, 129, 152, 191, 192, 195, 197, 198, 199, + /* 70 */ 200, 201, 202, 203, 204, 205, 207, 208, 209, 210, + /* 80 */ 211, 212, 213, 214, 215, 217, 218, 219, 220, 221, + /* 90 */ 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, + /* 100 */ 233, 234, 235, 236, 158, 237, 239, 171, 180, 184, + /* 110 */ 242, 194, 240, 246, 248, 250, 256, 249, 257, 260, + /* 120 */ 261, 258, 259, 262, 263, 266, 268, 270, 273, 276, + /* 130 */ 275, 278, 280, 274, 277, 279, 288, 291, 297, 286, + /* 140 */ 301, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 662, 716, 705, 713, 890, 890, 662, 662, 662, 662, - /* 10 */ 662, 662, 815, 680, 890, 662, 662, 662, 662, 662, - /* 20 */ 662, 662, 713, 662, 662, 718, 718, 718, 810, 662, - /* 30 */ 662, 662, 662, 662, 662, 662, 662, 662, 662, 662, - /* 40 */ 662, 662, 662, 662, 662, 662, 662, 662, 662, 662, - /* 50 */ 662, 662, 662, 662, 662, 662, 662, 662, 817, 819, - /* 60 */ 662, 837, 837, 808, 662, 662, 662, 662, 662, 662, - /* 70 */ 662, 662, 662, 662, 662, 662, 662, 662, 662, 662, - /* 80 */ 662, 662, 662, 703, 662, 701, 662, 662, 662, 662, - /* 90 */ 662, 662, 662, 662, 662, 662, 662, 662, 690, 662, - /* 100 */ 662, 662, 662, 662, 662, 682, 682, 662, 662, 662, - /* 110 */ 682, 844, 848, 842, 830, 838, 829, 825, 824, 852, - /* 120 */ 662, 682, 682, 682, 713, 682, 734, 732, 730, 722, - /* 130 */ 728, 724, 726, 720, 682, 711, 682, 711, 682, 751, - /* 140 */ 766, 662, 853, 889, 843, 879, 878, 885, 877, 876, - /* 150 */ 662, 872, 873, 875, 874, 662, 662, 662, 662, 881, - /* 160 */ 880, 662, 662, 662, 662, 662, 662, 662, 662, 662, - /* 170 */ 662, 662, 855, 662, 849, 845, 662, 662, 662, 662, - /* 180 */ 662, 662, 662, 662, 662, 662, 662, 662, 662, 662, - /* 190 */ 662, 662, 662, 662, 807, 662, 662, 816, 662, 662, - /* 200 */ 662, 662, 662, 662, 839, 662, 831, 662, 662, 662, - /* 210 */ 662, 662, 784, 662, 662, 662, 662, 662, 662, 662, - /* 220 */ 662, 662, 662, 662, 662, 662, 662, 662, 662, 894, - /* 230 */ 662, 662, 662, 775, 892, 662, 662, 662, 662, 662, - /* 240 */ 662, 662, 662, 662, 662, 662, 662, 662, 662, 662, - /* 250 */ 737, 662, 688, 686, 662, 678, 662, + /* 0 */ 669, 723, 712, 720, 900, 900, 669, 669, 669, 669, + /* 10 */ 669, 669, 825, 687, 900, 669, 669, 669, 669, 669, + /* 20 */ 669, 669, 720, 669, 669, 725, 725, 725, 820, 669, + /* 30 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 40 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 50 */ 669, 669, 669, 669, 669, 669, 669, 669, 827, 829, + /* 60 */ 669, 847, 847, 818, 669, 669, 669, 669, 669, 669, + /* 70 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 80 */ 669, 669, 669, 710, 669, 708, 669, 669, 669, 669, + /* 90 */ 669, 669, 669, 669, 669, 669, 669, 669, 697, 669, + /* 100 */ 669, 669, 669, 669, 669, 689, 689, 669, 669, 669, + /* 110 */ 689, 854, 858, 852, 840, 848, 839, 835, 834, 862, + /* 120 */ 669, 689, 689, 689, 720, 689, 741, 739, 737, 729, + /* 130 */ 735, 731, 733, 727, 689, 718, 689, 718, 689, 759, + /* 140 */ 775, 669, 863, 899, 853, 889, 888, 895, 887, 886, + /* 150 */ 669, 882, 883, 885, 884, 669, 669, 669, 669, 891, + /* 160 */ 890, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 170 */ 669, 669, 865, 669, 859, 855, 669, 669, 669, 669, + /* 180 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 190 */ 669, 669, 669, 669, 817, 669, 669, 826, 669, 669, + /* 200 */ 669, 669, 669, 669, 849, 669, 841, 669, 669, 669, + /* 210 */ 669, 669, 794, 669, 669, 669, 669, 669, 669, 669, + /* 220 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 904, + /* 230 */ 669, 669, 669, 785, 902, 669, 669, 669, 669, 669, + /* 240 */ 669, 669, 669, 669, 669, 669, 669, 669, 669, 669, + /* 250 */ 669, 744, 669, 695, 693, 669, 685, 669, }; /********** End of lemon-generated parsing tables *****************************/ @@ -541,6 +543,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* COMP => nothing */ 0, /* PRECISION => nothing */ 0, /* UPDATE => nothing */ + 0, /* CACHELAST => nothing */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* TAGS => nothing */ @@ -834,182 +837,184 @@ static const char *const yyTokenName[] = { /* 96 */ "COMP", /* 97 */ "PRECISION", /* 98 */ "UPDATE", - /* 99 */ "LP", - /* 100 */ "RP", - /* 101 */ "TAGS", - /* 102 */ "USING", - /* 103 */ "AS", - /* 104 */ "COMMA", - /* 105 */ "NULL", - /* 106 */ "SELECT", - /* 107 */ "UNION", - /* 108 */ "ALL", - /* 109 */ "FROM", - /* 110 */ "VARIABLE", - /* 111 */ "INTERVAL", - /* 112 */ "FILL", - /* 113 */ "SLIDING", - /* 114 */ "ORDER", - /* 115 */ "BY", - /* 116 */ "ASC", - /* 117 */ "DESC", - /* 118 */ "GROUP", - /* 119 */ "HAVING", - /* 120 */ "LIMIT", - /* 121 */ "OFFSET", - /* 122 */ "SLIMIT", - /* 123 */ "SOFFSET", - /* 124 */ "WHERE", - /* 125 */ "NOW", - /* 126 */ "RESET", - /* 127 */ "QUERY", - /* 128 */ "ADD", - /* 129 */ "COLUMN", - /* 130 */ "TAG", - /* 131 */ "CHANGE", - /* 132 */ "SET", - /* 133 */ "KILL", - /* 134 */ "CONNECTION", - /* 135 */ "STREAM", - /* 136 */ "COLON", - /* 137 */ "ABORT", - /* 138 */ "AFTER", - /* 139 */ "ATTACH", - /* 140 */ "BEFORE", - /* 141 */ "BEGIN", - /* 142 */ "CASCADE", - /* 143 */ "CLUSTER", - /* 144 */ "CONFLICT", - /* 145 */ "COPY", - /* 146 */ "DEFERRED", - /* 147 */ "DELIMITERS", - /* 148 */ "DETACH", - /* 149 */ "EACH", - /* 150 */ "END", - /* 151 */ "EXPLAIN", - /* 152 */ "FAIL", - /* 153 */ "FOR", - /* 154 */ "IGNORE", - /* 155 */ "IMMEDIATE", - /* 156 */ "INITIALLY", - /* 157 */ "INSTEAD", - /* 158 */ "MATCH", - /* 159 */ "KEY", - /* 160 */ "OF", - /* 161 */ "RAISE", - /* 162 */ "REPLACE", - /* 163 */ "RESTRICT", - /* 164 */ "ROW", - /* 165 */ "STATEMENT", - /* 166 */ "TRIGGER", - /* 167 */ "VIEW", - /* 168 */ "COUNT", - /* 169 */ "SUM", - /* 170 */ "AVG", - /* 171 */ "MIN", - /* 172 */ "MAX", - /* 173 */ "FIRST", - /* 174 */ "LAST", - /* 175 */ "TOP", - /* 176 */ "BOTTOM", - /* 177 */ "STDDEV", - /* 178 */ "PERCENTILE", - /* 179 */ "APERCENTILE", - /* 180 */ "LEASTSQUARES", - /* 181 */ "HISTOGRAM", - /* 182 */ "DIFF", - /* 183 */ "SPREAD", - /* 184 */ "TWA", - /* 185 */ "INTERP", - /* 186 */ "LAST_ROW", - /* 187 */ "RATE", - /* 188 */ "IRATE", - /* 189 */ "SUM_RATE", - /* 190 */ "SUM_IRATE", - /* 191 */ "AVG_RATE", - /* 192 */ "AVG_IRATE", - /* 193 */ "TBID", - /* 194 */ "SEMI", - /* 195 */ "NONE", - /* 196 */ "PREV", - /* 197 */ "LINEAR", - /* 198 */ "IMPORT", - /* 199 */ "METRIC", - /* 200 */ "TBNAME", - /* 201 */ "JOIN", - /* 202 */ "METRICS", - /* 203 */ "STABLE", - /* 204 */ "INSERT", - /* 205 */ "INTO", - /* 206 */ "VALUES", - /* 207 */ "error", - /* 208 */ "program", - /* 209 */ "cmd", - /* 210 */ "dbPrefix", - /* 211 */ "ids", - /* 212 */ "cpxName", - /* 213 */ "ifexists", - /* 214 */ "alter_db_optr", - /* 215 */ "acct_optr", - /* 216 */ "ifnotexists", - /* 217 */ "db_optr", - /* 218 */ "pps", - /* 219 */ "tseries", - /* 220 */ "dbs", - /* 221 */ "streams", - /* 222 */ "storage", - /* 223 */ "qtime", - /* 224 */ "users", - /* 225 */ "conns", - /* 226 */ "state", - /* 227 */ "keep", - /* 228 */ "tagitemlist", - /* 229 */ "cache", - /* 230 */ "replica", - /* 231 */ "quorum", - /* 232 */ "days", - /* 233 */ "minrows", - /* 234 */ "maxrows", - /* 235 */ "blocks", - /* 236 */ "ctime", - /* 237 */ "wal", - /* 238 */ "fsync", - /* 239 */ "comp", - /* 240 */ "prec", - /* 241 */ "update", - /* 242 */ "typename", - /* 243 */ "signed", - /* 244 */ "create_table_args", - /* 245 */ "create_table_list", - /* 246 */ "create_from_stable", - /* 247 */ "columnlist", - /* 248 */ "select", - /* 249 */ "column", - /* 250 */ "tagitem", - /* 251 */ "selcollist", - /* 252 */ "from", - /* 253 */ "where_opt", - /* 254 */ "interval_opt", - /* 255 */ "fill_opt", - /* 256 */ "sliding_opt", - /* 257 */ "groupby_opt", - /* 258 */ "orderby_opt", - /* 259 */ "having_opt", - /* 260 */ "slimit_opt", - /* 261 */ "limit_opt", - /* 262 */ "union", - /* 263 */ "sclp", - /* 264 */ "expr", - /* 265 */ "as", - /* 266 */ "tablelist", - /* 267 */ "tmvar", - /* 268 */ "sortlist", - /* 269 */ "sortitem", - /* 270 */ "item", - /* 271 */ "sortorder", - /* 272 */ "grouplist", - /* 273 */ "exprlist", - /* 274 */ "expritem", + /* 99 */ "CACHELAST", + /* 100 */ "LP", + /* 101 */ "RP", + /* 102 */ "TAGS", + /* 103 */ "USING", + /* 104 */ "AS", + /* 105 */ "COMMA", + /* 106 */ "NULL", + /* 107 */ "SELECT", + /* 108 */ "UNION", + /* 109 */ "ALL", + /* 110 */ "FROM", + /* 111 */ "VARIABLE", + /* 112 */ "INTERVAL", + /* 113 */ "FILL", + /* 114 */ "SLIDING", + /* 115 */ "ORDER", + /* 116 */ "BY", + /* 117 */ "ASC", + /* 118 */ "DESC", + /* 119 */ "GROUP", + /* 120 */ "HAVING", + /* 121 */ "LIMIT", + /* 122 */ "OFFSET", + /* 123 */ "SLIMIT", + /* 124 */ "SOFFSET", + /* 125 */ "WHERE", + /* 126 */ "NOW", + /* 127 */ "RESET", + /* 128 */ "QUERY", + /* 129 */ "ADD", + /* 130 */ "COLUMN", + /* 131 */ "TAG", + /* 132 */ "CHANGE", + /* 133 */ "SET", + /* 134 */ "KILL", + /* 135 */ "CONNECTION", + /* 136 */ "STREAM", + /* 137 */ "COLON", + /* 138 */ "ABORT", + /* 139 */ "AFTER", + /* 140 */ "ATTACH", + /* 141 */ "BEFORE", + /* 142 */ "BEGIN", + /* 143 */ "CASCADE", + /* 144 */ "CLUSTER", + /* 145 */ "CONFLICT", + /* 146 */ "COPY", + /* 147 */ "DEFERRED", + /* 148 */ "DELIMITERS", + /* 149 */ "DETACH", + /* 150 */ "EACH", + /* 151 */ "END", + /* 152 */ "EXPLAIN", + /* 153 */ "FAIL", + /* 154 */ "FOR", + /* 155 */ "IGNORE", + /* 156 */ "IMMEDIATE", + /* 157 */ "INITIALLY", + /* 158 */ "INSTEAD", + /* 159 */ "MATCH", + /* 160 */ "KEY", + /* 161 */ "OF", + /* 162 */ "RAISE", + /* 163 */ "REPLACE", + /* 164 */ "RESTRICT", + /* 165 */ "ROW", + /* 166 */ "STATEMENT", + /* 167 */ "TRIGGER", + /* 168 */ "VIEW", + /* 169 */ "COUNT", + /* 170 */ "SUM", + /* 171 */ "AVG", + /* 172 */ "MIN", + /* 173 */ "MAX", + /* 174 */ "FIRST", + /* 175 */ "LAST", + /* 176 */ "TOP", + /* 177 */ "BOTTOM", + /* 178 */ "STDDEV", + /* 179 */ "PERCENTILE", + /* 180 */ "APERCENTILE", + /* 181 */ "LEASTSQUARES", + /* 182 */ "HISTOGRAM", + /* 183 */ "DIFF", + /* 184 */ "SPREAD", + /* 185 */ "TWA", + /* 186 */ "INTERP", + /* 187 */ "LAST_ROW", + /* 188 */ "RATE", + /* 189 */ "IRATE", + /* 190 */ "SUM_RATE", + /* 191 */ "SUM_IRATE", + /* 192 */ "AVG_RATE", + /* 193 */ "AVG_IRATE", + /* 194 */ "TBID", + /* 195 */ "SEMI", + /* 196 */ "NONE", + /* 197 */ "PREV", + /* 198 */ "LINEAR", + /* 199 */ "IMPORT", + /* 200 */ "METRIC", + /* 201 */ "TBNAME", + /* 202 */ "JOIN", + /* 203 */ "METRICS", + /* 204 */ "STABLE", + /* 205 */ "INSERT", + /* 206 */ "INTO", + /* 207 */ "VALUES", + /* 208 */ "error", + /* 209 */ "program", + /* 210 */ "cmd", + /* 211 */ "dbPrefix", + /* 212 */ "ids", + /* 213 */ "cpxName", + /* 214 */ "ifexists", + /* 215 */ "alter_db_optr", + /* 216 */ "acct_optr", + /* 217 */ "ifnotexists", + /* 218 */ "db_optr", + /* 219 */ "pps", + /* 220 */ "tseries", + /* 221 */ "dbs", + /* 222 */ "streams", + /* 223 */ "storage", + /* 224 */ "qtime", + /* 225 */ "users", + /* 226 */ "conns", + /* 227 */ "state", + /* 228 */ "keep", + /* 229 */ "tagitemlist", + /* 230 */ "cache", + /* 231 */ "replica", + /* 232 */ "quorum", + /* 233 */ "days", + /* 234 */ "minrows", + /* 235 */ "maxrows", + /* 236 */ "blocks", + /* 237 */ "ctime", + /* 238 */ "wal", + /* 239 */ "fsync", + /* 240 */ "comp", + /* 241 */ "prec", + /* 242 */ "update", + /* 243 */ "cachelast", + /* 244 */ "typename", + /* 245 */ "signed", + /* 246 */ "create_table_args", + /* 247 */ "create_table_list", + /* 248 */ "create_from_stable", + /* 249 */ "columnlist", + /* 250 */ "select", + /* 251 */ "column", + /* 252 */ "tagitem", + /* 253 */ "selcollist", + /* 254 */ "from", + /* 255 */ "where_opt", + /* 256 */ "interval_opt", + /* 257 */ "fill_opt", + /* 258 */ "sliding_opt", + /* 259 */ "groupby_opt", + /* 260 */ "orderby_opt", + /* 261 */ "having_opt", + /* 262 */ "slimit_opt", + /* 263 */ "limit_opt", + /* 264 */ "union", + /* 265 */ "sclp", + /* 266 */ "expr", + /* 267 */ "as", + /* 268 */ "tablelist", + /* 269 */ "tmvar", + /* 270 */ "sortlist", + /* 271 */ "sortitem", + /* 272 */ "item", + /* 273 */ "sortorder", + /* 274 */ "grouplist", + /* 275 */ "exprlist", + /* 276 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1103,156 +1108,159 @@ static const char *const yyRuleName[] = { /* 83 */ "comp ::= COMP INTEGER", /* 84 */ "prec ::= PRECISION STRING", /* 85 */ "update ::= UPDATE INTEGER", - /* 86 */ "db_optr ::=", - /* 87 */ "db_optr ::= db_optr cache", - /* 88 */ "db_optr ::= db_optr replica", - /* 89 */ "db_optr ::= db_optr quorum", - /* 90 */ "db_optr ::= db_optr days", - /* 91 */ "db_optr ::= db_optr minrows", - /* 92 */ "db_optr ::= db_optr maxrows", - /* 93 */ "db_optr ::= db_optr blocks", - /* 94 */ "db_optr ::= db_optr ctime", - /* 95 */ "db_optr ::= db_optr wal", - /* 96 */ "db_optr ::= db_optr fsync", - /* 97 */ "db_optr ::= db_optr comp", - /* 98 */ "db_optr ::= db_optr prec", - /* 99 */ "db_optr ::= db_optr keep", - /* 100 */ "db_optr ::= db_optr update", - /* 101 */ "alter_db_optr ::=", - /* 102 */ "alter_db_optr ::= alter_db_optr replica", - /* 103 */ "alter_db_optr ::= alter_db_optr quorum", - /* 104 */ "alter_db_optr ::= alter_db_optr keep", - /* 105 */ "alter_db_optr ::= alter_db_optr blocks", - /* 106 */ "alter_db_optr ::= alter_db_optr comp", - /* 107 */ "alter_db_optr ::= alter_db_optr wal", - /* 108 */ "alter_db_optr ::= alter_db_optr fsync", - /* 109 */ "alter_db_optr ::= alter_db_optr update", - /* 110 */ "typename ::= ids", - /* 111 */ "typename ::= ids LP signed RP", - /* 112 */ "signed ::= INTEGER", - /* 113 */ "signed ::= PLUS INTEGER", - /* 114 */ "signed ::= MINUS INTEGER", - /* 115 */ "cmd ::= CREATE TABLE create_table_args", - /* 116 */ "cmd ::= CREATE TABLE create_table_list", - /* 117 */ "create_table_list ::= create_from_stable", - /* 118 */ "create_table_list ::= create_table_list create_from_stable", - /* 119 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", - /* 120 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", - /* 121 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", - /* 122 */ "create_table_args ::= ifnotexists ids cpxName AS select", - /* 123 */ "columnlist ::= columnlist COMMA column", - /* 124 */ "columnlist ::= column", - /* 125 */ "column ::= ids typename", - /* 126 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 127 */ "tagitemlist ::= tagitem", - /* 128 */ "tagitem ::= INTEGER", - /* 129 */ "tagitem ::= FLOAT", - /* 130 */ "tagitem ::= STRING", - /* 131 */ "tagitem ::= BOOL", - /* 132 */ "tagitem ::= NULL", - /* 133 */ "tagitem ::= MINUS INTEGER", - /* 134 */ "tagitem ::= MINUS FLOAT", - /* 135 */ "tagitem ::= PLUS INTEGER", - /* 136 */ "tagitem ::= PLUS FLOAT", - /* 137 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 138 */ "union ::= select", - /* 139 */ "union ::= LP union RP", - /* 140 */ "union ::= union UNION ALL select", - /* 141 */ "union ::= union UNION ALL LP select RP", - /* 142 */ "cmd ::= union", - /* 143 */ "select ::= SELECT selcollist", - /* 144 */ "sclp ::= selcollist COMMA", - /* 145 */ "sclp ::=", - /* 146 */ "selcollist ::= sclp expr as", - /* 147 */ "selcollist ::= sclp STAR", - /* 148 */ "as ::= AS ids", - /* 149 */ "as ::= ids", - /* 150 */ "as ::=", - /* 151 */ "from ::= FROM tablelist", - /* 152 */ "tablelist ::= ids cpxName", - /* 153 */ "tablelist ::= ids cpxName ids", - /* 154 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 155 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 156 */ "tmvar ::= VARIABLE", - /* 157 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 158 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 159 */ "interval_opt ::=", - /* 160 */ "fill_opt ::=", - /* 161 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 162 */ "fill_opt ::= FILL LP ID RP", - /* 163 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 164 */ "sliding_opt ::=", - /* 165 */ "orderby_opt ::=", - /* 166 */ "orderby_opt ::= ORDER BY sortlist", - /* 167 */ "sortlist ::= sortlist COMMA item sortorder", - /* 168 */ "sortlist ::= item sortorder", - /* 169 */ "item ::= ids cpxName", - /* 170 */ "sortorder ::= ASC", - /* 171 */ "sortorder ::= DESC", - /* 172 */ "sortorder ::=", - /* 173 */ "groupby_opt ::=", - /* 174 */ "groupby_opt ::= GROUP BY grouplist", - /* 175 */ "grouplist ::= grouplist COMMA item", - /* 176 */ "grouplist ::= item", - /* 177 */ "having_opt ::=", - /* 178 */ "having_opt ::= HAVING expr", - /* 179 */ "limit_opt ::=", - /* 180 */ "limit_opt ::= LIMIT signed", - /* 181 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 182 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 183 */ "slimit_opt ::=", - /* 184 */ "slimit_opt ::= SLIMIT signed", - /* 185 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 186 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 187 */ "where_opt ::=", - /* 188 */ "where_opt ::= WHERE expr", - /* 189 */ "expr ::= LP expr RP", - /* 190 */ "expr ::= ID", - /* 191 */ "expr ::= ID DOT ID", - /* 192 */ "expr ::= ID DOT STAR", - /* 193 */ "expr ::= INTEGER", - /* 194 */ "expr ::= MINUS INTEGER", - /* 195 */ "expr ::= PLUS INTEGER", - /* 196 */ "expr ::= FLOAT", - /* 197 */ "expr ::= MINUS FLOAT", - /* 198 */ "expr ::= PLUS FLOAT", - /* 199 */ "expr ::= STRING", - /* 200 */ "expr ::= NOW", - /* 201 */ "expr ::= VARIABLE", - /* 202 */ "expr ::= BOOL", - /* 203 */ "expr ::= ID LP exprlist RP", - /* 204 */ "expr ::= ID LP STAR RP", - /* 205 */ "expr ::= expr IS NULL", - /* 206 */ "expr ::= expr IS NOT NULL", - /* 207 */ "expr ::= expr LT expr", - /* 208 */ "expr ::= expr GT expr", - /* 209 */ "expr ::= expr LE expr", - /* 210 */ "expr ::= expr GE expr", - /* 211 */ "expr ::= expr NE expr", - /* 212 */ "expr ::= expr EQ expr", - /* 213 */ "expr ::= expr AND expr", - /* 214 */ "expr ::= expr OR expr", - /* 215 */ "expr ::= expr PLUS expr", - /* 216 */ "expr ::= expr MINUS expr", - /* 217 */ "expr ::= expr STAR expr", - /* 218 */ "expr ::= expr SLASH expr", - /* 219 */ "expr ::= expr REM expr", - /* 220 */ "expr ::= expr LIKE expr", - /* 221 */ "expr ::= expr IN LP exprlist RP", - /* 222 */ "exprlist ::= exprlist COMMA expritem", - /* 223 */ "exprlist ::= expritem", - /* 224 */ "expritem ::= expr", - /* 225 */ "expritem ::=", - /* 226 */ "cmd ::= RESET QUERY CACHE", - /* 227 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 228 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 229 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 230 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 231 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 232 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 233 */ "cmd ::= KILL CONNECTION INTEGER", - /* 234 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 235 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 86 */ "cachelast ::= CACHELAST INTEGER", + /* 87 */ "db_optr ::=", + /* 88 */ "db_optr ::= db_optr cache", + /* 89 */ "db_optr ::= db_optr replica", + /* 90 */ "db_optr ::= db_optr quorum", + /* 91 */ "db_optr ::= db_optr days", + /* 92 */ "db_optr ::= db_optr minrows", + /* 93 */ "db_optr ::= db_optr maxrows", + /* 94 */ "db_optr ::= db_optr blocks", + /* 95 */ "db_optr ::= db_optr ctime", + /* 96 */ "db_optr ::= db_optr wal", + /* 97 */ "db_optr ::= db_optr fsync", + /* 98 */ "db_optr ::= db_optr comp", + /* 99 */ "db_optr ::= db_optr prec", + /* 100 */ "db_optr ::= db_optr keep", + /* 101 */ "db_optr ::= db_optr update", + /* 102 */ "db_optr ::= db_optr cachelast", + /* 103 */ "alter_db_optr ::=", + /* 104 */ "alter_db_optr ::= alter_db_optr replica", + /* 105 */ "alter_db_optr ::= alter_db_optr quorum", + /* 106 */ "alter_db_optr ::= alter_db_optr keep", + /* 107 */ "alter_db_optr ::= alter_db_optr blocks", + /* 108 */ "alter_db_optr ::= alter_db_optr comp", + /* 109 */ "alter_db_optr ::= alter_db_optr wal", + /* 110 */ "alter_db_optr ::= alter_db_optr fsync", + /* 111 */ "alter_db_optr ::= alter_db_optr update", + /* 112 */ "alter_db_optr ::= alter_db_optr cachelast", + /* 113 */ "typename ::= ids", + /* 114 */ "typename ::= ids LP signed RP", + /* 115 */ "signed ::= INTEGER", + /* 116 */ "signed ::= PLUS INTEGER", + /* 117 */ "signed ::= MINUS INTEGER", + /* 118 */ "cmd ::= CREATE TABLE create_table_args", + /* 119 */ "cmd ::= CREATE TABLE create_table_list", + /* 120 */ "create_table_list ::= create_from_stable", + /* 121 */ "create_table_list ::= create_table_list create_from_stable", + /* 122 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", + /* 123 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", + /* 124 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", + /* 125 */ "create_table_args ::= ifnotexists ids cpxName AS select", + /* 126 */ "columnlist ::= columnlist COMMA column", + /* 127 */ "columnlist ::= column", + /* 128 */ "column ::= ids typename", + /* 129 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 130 */ "tagitemlist ::= tagitem", + /* 131 */ "tagitem ::= INTEGER", + /* 132 */ "tagitem ::= FLOAT", + /* 133 */ "tagitem ::= STRING", + /* 134 */ "tagitem ::= BOOL", + /* 135 */ "tagitem ::= NULL", + /* 136 */ "tagitem ::= MINUS INTEGER", + /* 137 */ "tagitem ::= MINUS FLOAT", + /* 138 */ "tagitem ::= PLUS INTEGER", + /* 139 */ "tagitem ::= PLUS FLOAT", + /* 140 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 141 */ "union ::= select", + /* 142 */ "union ::= LP union RP", + /* 143 */ "union ::= union UNION ALL select", + /* 144 */ "union ::= union UNION ALL LP select RP", + /* 145 */ "cmd ::= union", + /* 146 */ "select ::= SELECT selcollist", + /* 147 */ "sclp ::= selcollist COMMA", + /* 148 */ "sclp ::=", + /* 149 */ "selcollist ::= sclp expr as", + /* 150 */ "selcollist ::= sclp STAR", + /* 151 */ "as ::= AS ids", + /* 152 */ "as ::= ids", + /* 153 */ "as ::=", + /* 154 */ "from ::= FROM tablelist", + /* 155 */ "tablelist ::= ids cpxName", + /* 156 */ "tablelist ::= ids cpxName ids", + /* 157 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 158 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 159 */ "tmvar ::= VARIABLE", + /* 160 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 161 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 162 */ "interval_opt ::=", + /* 163 */ "fill_opt ::=", + /* 164 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 165 */ "fill_opt ::= FILL LP ID RP", + /* 166 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 167 */ "sliding_opt ::=", + /* 168 */ "orderby_opt ::=", + /* 169 */ "orderby_opt ::= ORDER BY sortlist", + /* 170 */ "sortlist ::= sortlist COMMA item sortorder", + /* 171 */ "sortlist ::= item sortorder", + /* 172 */ "item ::= ids cpxName", + /* 173 */ "sortorder ::= ASC", + /* 174 */ "sortorder ::= DESC", + /* 175 */ "sortorder ::=", + /* 176 */ "groupby_opt ::=", + /* 177 */ "groupby_opt ::= GROUP BY grouplist", + /* 178 */ "grouplist ::= grouplist COMMA item", + /* 179 */ "grouplist ::= item", + /* 180 */ "having_opt ::=", + /* 181 */ "having_opt ::= HAVING expr", + /* 182 */ "limit_opt ::=", + /* 183 */ "limit_opt ::= LIMIT signed", + /* 184 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 185 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 186 */ "slimit_opt ::=", + /* 187 */ "slimit_opt ::= SLIMIT signed", + /* 188 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 189 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 190 */ "where_opt ::=", + /* 191 */ "where_opt ::= WHERE expr", + /* 192 */ "expr ::= LP expr RP", + /* 193 */ "expr ::= ID", + /* 194 */ "expr ::= ID DOT ID", + /* 195 */ "expr ::= ID DOT STAR", + /* 196 */ "expr ::= INTEGER", + /* 197 */ "expr ::= MINUS INTEGER", + /* 198 */ "expr ::= PLUS INTEGER", + /* 199 */ "expr ::= FLOAT", + /* 200 */ "expr ::= MINUS FLOAT", + /* 201 */ "expr ::= PLUS FLOAT", + /* 202 */ "expr ::= STRING", + /* 203 */ "expr ::= NOW", + /* 204 */ "expr ::= VARIABLE", + /* 205 */ "expr ::= BOOL", + /* 206 */ "expr ::= ID LP exprlist RP", + /* 207 */ "expr ::= ID LP STAR RP", + /* 208 */ "expr ::= expr IS NULL", + /* 209 */ "expr ::= expr IS NOT NULL", + /* 210 */ "expr ::= expr LT expr", + /* 211 */ "expr ::= expr GT expr", + /* 212 */ "expr ::= expr LE expr", + /* 213 */ "expr ::= expr GE expr", + /* 214 */ "expr ::= expr NE expr", + /* 215 */ "expr ::= expr EQ expr", + /* 216 */ "expr ::= expr AND expr", + /* 217 */ "expr ::= expr OR expr", + /* 218 */ "expr ::= expr PLUS expr", + /* 219 */ "expr ::= expr MINUS expr", + /* 220 */ "expr ::= expr STAR expr", + /* 221 */ "expr ::= expr SLASH expr", + /* 222 */ "expr ::= expr REM expr", + /* 223 */ "expr ::= expr LIKE expr", + /* 224 */ "expr ::= expr IN LP exprlist RP", + /* 225 */ "exprlist ::= exprlist COMMA expritem", + /* 226 */ "exprlist ::= expritem", + /* 227 */ "expritem ::= expr", + /* 228 */ "expritem ::=", + /* 229 */ "cmd ::= RESET QUERY CACHE", + /* 230 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 231 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 232 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 233 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 234 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 235 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 236 */ "cmd ::= KILL CONNECTION INTEGER", + /* 237 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 238 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1373,51 +1381,51 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 227: /* keep */ - case 228: /* tagitemlist */ - case 247: /* columnlist */ - case 255: /* fill_opt */ - case 257: /* groupby_opt */ - case 258: /* orderby_opt */ - case 268: /* sortlist */ - case 272: /* grouplist */ + case 228: /* keep */ + case 229: /* tagitemlist */ + case 249: /* columnlist */ + case 257: /* fill_opt */ + case 259: /* groupby_opt */ + case 260: /* orderby_opt */ + case 270: /* sortlist */ + case 274: /* grouplist */ { -taosArrayDestroy((yypminor->yy131)); +taosArrayDestroy((yypminor->yy403)); } break; - case 245: /* create_table_list */ + case 247: /* create_table_list */ { -destroyCreateTableSql((yypminor->yy538)); +destroyCreateTableSql((yypminor->yy436)); } break; - case 248: /* select */ + case 250: /* select */ { -doDestroyQuerySql((yypminor->yy84)); +doDestroyQuerySql((yypminor->yy4)); } break; - case 251: /* selcollist */ - case 263: /* sclp */ - case 273: /* exprlist */ + case 253: /* selcollist */ + case 265: /* sclp */ + case 275: /* exprlist */ { - tSqlExprListDestroy((yypminor->yy478)); +tSqlExprListDestroy((yypminor->yy382)); } break; - case 253: /* where_opt */ - case 259: /* having_opt */ - case 264: /* expr */ - case 274: /* expritem */ + case 255: /* where_opt */ + case 261: /* having_opt */ + case 266: /* expr */ + case 276: /* expritem */ { - tSqlExprDestroy((yypminor->yy420)); +tSqlExprDestroy((yypminor->yy522)); } break; - case 262: /* union */ + case 264: /* union */ { -destroyAllSelectClause((yypminor->yy513)); +destroyAllSelectClause((yypminor->yy13)); } break; - case 269: /* sortitem */ + case 271: /* sortitem */ { -tVariantDestroy(&(yypminor->yy516)); +tVariantDestroy(&(yypminor->yy488)); } break; /********* End destructor definitions *****************************************/ @@ -1711,242 +1719,245 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 208, -1 }, /* (0) program ::= cmd */ - { 209, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 209, -2 }, /* (2) cmd ::= SHOW MNODES */ - { 209, -2 }, /* (3) cmd ::= SHOW DNODES */ - { 209, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ - { 209, -2 }, /* (5) cmd ::= SHOW USERS */ - { 209, -2 }, /* (6) cmd ::= SHOW MODULES */ - { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ - { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ - { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ - { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ - { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ - { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ - { 209, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - { 210, 0 }, /* (15) dbPrefix ::= */ - { 210, -2 }, /* (16) dbPrefix ::= ids DOT */ - { 212, 0 }, /* (17) cpxName ::= */ - { 212, -2 }, /* (18) cpxName ::= DOT ids */ - { 209, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ - { 209, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ - { 209, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ - { 209, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 209, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ - { 209, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 209, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ - { 209, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 209, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ - { 209, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ - { 209, -3 }, /* (29) cmd ::= DROP DNODE ids */ - { 209, -3 }, /* (30) cmd ::= DROP USER ids */ - { 209, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ - { 209, -2 }, /* (32) cmd ::= USE ids */ - { 209, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ - { 209, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ - { 209, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 209, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ - { 209, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ - { 209, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ - { 209, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ - { 209, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 209, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 209, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 211, -1 }, /* (43) ids ::= ID */ - { 211, -1 }, /* (44) ids ::= STRING */ - { 213, -2 }, /* (45) ifexists ::= IF EXISTS */ - { 213, 0 }, /* (46) ifexists ::= */ - { 216, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ - { 216, 0 }, /* (48) ifnotexists ::= */ - { 209, -3 }, /* (49) cmd ::= CREATE DNODE ids */ - { 209, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 209, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 209, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ - { 218, 0 }, /* (53) pps ::= */ - { 218, -2 }, /* (54) pps ::= PPS INTEGER */ - { 219, 0 }, /* (55) tseries ::= */ - { 219, -2 }, /* (56) tseries ::= TSERIES INTEGER */ - { 220, 0 }, /* (57) dbs ::= */ - { 220, -2 }, /* (58) dbs ::= DBS INTEGER */ - { 221, 0 }, /* (59) streams ::= */ - { 221, -2 }, /* (60) streams ::= STREAMS INTEGER */ - { 222, 0 }, /* (61) storage ::= */ - { 222, -2 }, /* (62) storage ::= STORAGE INTEGER */ - { 223, 0 }, /* (63) qtime ::= */ - { 223, -2 }, /* (64) qtime ::= QTIME INTEGER */ - { 224, 0 }, /* (65) users ::= */ - { 224, -2 }, /* (66) users ::= USERS INTEGER */ - { 225, 0 }, /* (67) conns ::= */ - { 225, -2 }, /* (68) conns ::= CONNS INTEGER */ - { 226, 0 }, /* (69) state ::= */ - { 226, -2 }, /* (70) state ::= STATE ids */ - { 215, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 227, -2 }, /* (72) keep ::= KEEP tagitemlist */ - { 229, -2 }, /* (73) cache ::= CACHE INTEGER */ - { 230, -2 }, /* (74) replica ::= REPLICA INTEGER */ - { 231, -2 }, /* (75) quorum ::= QUORUM INTEGER */ - { 232, -2 }, /* (76) days ::= DAYS INTEGER */ - { 233, -2 }, /* (77) minrows ::= MINROWS INTEGER */ - { 234, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ - { 235, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ - { 236, -2 }, /* (80) ctime ::= CTIME INTEGER */ - { 237, -2 }, /* (81) wal ::= WAL INTEGER */ - { 238, -2 }, /* (82) fsync ::= FSYNC INTEGER */ - { 239, -2 }, /* (83) comp ::= COMP INTEGER */ - { 240, -2 }, /* (84) prec ::= PRECISION STRING */ - { 241, -2 }, /* (85) update ::= UPDATE INTEGER */ - { 217, 0 }, /* (86) db_optr ::= */ - { 217, -2 }, /* (87) db_optr ::= db_optr cache */ - { 217, -2 }, /* (88) db_optr ::= db_optr replica */ - { 217, -2 }, /* (89) db_optr ::= db_optr quorum */ - { 217, -2 }, /* (90) db_optr ::= db_optr days */ - { 217, -2 }, /* (91) db_optr ::= db_optr minrows */ - { 217, -2 }, /* (92) db_optr ::= db_optr maxrows */ - { 217, -2 }, /* (93) db_optr ::= db_optr blocks */ - { 217, -2 }, /* (94) db_optr ::= db_optr ctime */ - { 217, -2 }, /* (95) db_optr ::= db_optr wal */ - { 217, -2 }, /* (96) db_optr ::= db_optr fsync */ - { 217, -2 }, /* (97) db_optr ::= db_optr comp */ - { 217, -2 }, /* (98) db_optr ::= db_optr prec */ - { 217, -2 }, /* (99) db_optr ::= db_optr keep */ - { 217, -2 }, /* (100) db_optr ::= db_optr update */ - { 214, 0 }, /* (101) alter_db_optr ::= */ - { 214, -2 }, /* (102) alter_db_optr ::= alter_db_optr replica */ - { 214, -2 }, /* (103) alter_db_optr ::= alter_db_optr quorum */ - { 214, -2 }, /* (104) alter_db_optr ::= alter_db_optr keep */ - { 214, -2 }, /* (105) alter_db_optr ::= alter_db_optr blocks */ - { 214, -2 }, /* (106) alter_db_optr ::= alter_db_optr comp */ - { 214, -2 }, /* (107) alter_db_optr ::= alter_db_optr wal */ - { 214, -2 }, /* (108) alter_db_optr ::= alter_db_optr fsync */ - { 214, -2 }, /* (109) alter_db_optr ::= alter_db_optr update */ - { 242, -1 }, /* (110) typename ::= ids */ - { 242, -4 }, /* (111) typename ::= ids LP signed RP */ - { 243, -1 }, /* (112) signed ::= INTEGER */ - { 243, -2 }, /* (113) signed ::= PLUS INTEGER */ - { 243, -2 }, /* (114) signed ::= MINUS INTEGER */ - { 209, -3 }, /* (115) cmd ::= CREATE TABLE create_table_args */ - { 209, -3 }, /* (116) cmd ::= CREATE TABLE create_table_list */ - { 245, -1 }, /* (117) create_table_list ::= create_from_stable */ - { 245, -2 }, /* (118) create_table_list ::= create_table_list create_from_stable */ - { 244, -6 }, /* (119) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - { 244, -10 }, /* (120) create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - { 246, -10 }, /* (121) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - { 244, -5 }, /* (122) create_table_args ::= ifnotexists ids cpxName AS select */ - { 247, -3 }, /* (123) columnlist ::= columnlist COMMA column */ - { 247, -1 }, /* (124) columnlist ::= column */ - { 249, -2 }, /* (125) column ::= ids typename */ - { 228, -3 }, /* (126) tagitemlist ::= tagitemlist COMMA tagitem */ - { 228, -1 }, /* (127) tagitemlist ::= tagitem */ - { 250, -1 }, /* (128) tagitem ::= INTEGER */ - { 250, -1 }, /* (129) tagitem ::= FLOAT */ - { 250, -1 }, /* (130) tagitem ::= STRING */ - { 250, -1 }, /* (131) tagitem ::= BOOL */ - { 250, -1 }, /* (132) tagitem ::= NULL */ - { 250, -2 }, /* (133) tagitem ::= MINUS INTEGER */ - { 250, -2 }, /* (134) tagitem ::= MINUS FLOAT */ - { 250, -2 }, /* (135) tagitem ::= PLUS INTEGER */ - { 250, -2 }, /* (136) tagitem ::= PLUS FLOAT */ - { 248, -12 }, /* (137) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - { 262, -1 }, /* (138) union ::= select */ - { 262, -3 }, /* (139) union ::= LP union RP */ - { 262, -4 }, /* (140) union ::= union UNION ALL select */ - { 262, -6 }, /* (141) union ::= union UNION ALL LP select RP */ - { 209, -1 }, /* (142) cmd ::= union */ - { 248, -2 }, /* (143) select ::= SELECT selcollist */ - { 263, -2 }, /* (144) sclp ::= selcollist COMMA */ - { 263, 0 }, /* (145) sclp ::= */ - { 251, -3 }, /* (146) selcollist ::= sclp expr as */ - { 251, -2 }, /* (147) selcollist ::= sclp STAR */ - { 265, -2 }, /* (148) as ::= AS ids */ - { 265, -1 }, /* (149) as ::= ids */ - { 265, 0 }, /* (150) as ::= */ - { 252, -2 }, /* (151) from ::= FROM tablelist */ - { 266, -2 }, /* (152) tablelist ::= ids cpxName */ - { 266, -3 }, /* (153) tablelist ::= ids cpxName ids */ - { 266, -4 }, /* (154) tablelist ::= tablelist COMMA ids cpxName */ - { 266, -5 }, /* (155) tablelist ::= tablelist COMMA ids cpxName ids */ - { 267, -1 }, /* (156) tmvar ::= VARIABLE */ - { 254, -4 }, /* (157) interval_opt ::= INTERVAL LP tmvar RP */ - { 254, -6 }, /* (158) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - { 254, 0 }, /* (159) interval_opt ::= */ - { 255, 0 }, /* (160) fill_opt ::= */ - { 255, -6 }, /* (161) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 255, -4 }, /* (162) fill_opt ::= FILL LP ID RP */ - { 256, -4 }, /* (163) sliding_opt ::= SLIDING LP tmvar RP */ - { 256, 0 }, /* (164) sliding_opt ::= */ - { 258, 0 }, /* (165) orderby_opt ::= */ - { 258, -3 }, /* (166) orderby_opt ::= ORDER BY sortlist */ - { 268, -4 }, /* (167) sortlist ::= sortlist COMMA item sortorder */ - { 268, -2 }, /* (168) sortlist ::= item sortorder */ - { 270, -2 }, /* (169) item ::= ids cpxName */ - { 271, -1 }, /* (170) sortorder ::= ASC */ - { 271, -1 }, /* (171) sortorder ::= DESC */ - { 271, 0 }, /* (172) sortorder ::= */ - { 257, 0 }, /* (173) groupby_opt ::= */ - { 257, -3 }, /* (174) groupby_opt ::= GROUP BY grouplist */ - { 272, -3 }, /* (175) grouplist ::= grouplist COMMA item */ - { 272, -1 }, /* (176) grouplist ::= item */ - { 259, 0 }, /* (177) having_opt ::= */ - { 259, -2 }, /* (178) having_opt ::= HAVING expr */ - { 261, 0 }, /* (179) limit_opt ::= */ - { 261, -2 }, /* (180) limit_opt ::= LIMIT signed */ - { 261, -4 }, /* (181) limit_opt ::= LIMIT signed OFFSET signed */ - { 261, -4 }, /* (182) limit_opt ::= LIMIT signed COMMA signed */ - { 260, 0 }, /* (183) slimit_opt ::= */ - { 260, -2 }, /* (184) slimit_opt ::= SLIMIT signed */ - { 260, -4 }, /* (185) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 260, -4 }, /* (186) slimit_opt ::= SLIMIT signed COMMA signed */ - { 253, 0 }, /* (187) where_opt ::= */ - { 253, -2 }, /* (188) where_opt ::= WHERE expr */ - { 264, -3 }, /* (189) expr ::= LP expr RP */ - { 264, -1 }, /* (190) expr ::= ID */ - { 264, -3 }, /* (191) expr ::= ID DOT ID */ - { 264, -3 }, /* (192) expr ::= ID DOT STAR */ - { 264, -1 }, /* (193) expr ::= INTEGER */ - { 264, -2 }, /* (194) expr ::= MINUS INTEGER */ - { 264, -2 }, /* (195) expr ::= PLUS INTEGER */ - { 264, -1 }, /* (196) expr ::= FLOAT */ - { 264, -2 }, /* (197) expr ::= MINUS FLOAT */ - { 264, -2 }, /* (198) expr ::= PLUS FLOAT */ - { 264, -1 }, /* (199) expr ::= STRING */ - { 264, -1 }, /* (200) expr ::= NOW */ - { 264, -1 }, /* (201) expr ::= VARIABLE */ - { 264, -1 }, /* (202) expr ::= BOOL */ - { 264, -4 }, /* (203) expr ::= ID LP exprlist RP */ - { 264, -4 }, /* (204) expr ::= ID LP STAR RP */ - { 264, -3 }, /* (205) expr ::= expr IS NULL */ - { 264, -4 }, /* (206) expr ::= expr IS NOT NULL */ - { 264, -3 }, /* (207) expr ::= expr LT expr */ - { 264, -3 }, /* (208) expr ::= expr GT expr */ - { 264, -3 }, /* (209) expr ::= expr LE expr */ - { 264, -3 }, /* (210) expr ::= expr GE expr */ - { 264, -3 }, /* (211) expr ::= expr NE expr */ - { 264, -3 }, /* (212) expr ::= expr EQ expr */ - { 264, -3 }, /* (213) expr ::= expr AND expr */ - { 264, -3 }, /* (214) expr ::= expr OR expr */ - { 264, -3 }, /* (215) expr ::= expr PLUS expr */ - { 264, -3 }, /* (216) expr ::= expr MINUS expr */ - { 264, -3 }, /* (217) expr ::= expr STAR expr */ - { 264, -3 }, /* (218) expr ::= expr SLASH expr */ - { 264, -3 }, /* (219) expr ::= expr REM expr */ - { 264, -3 }, /* (220) expr ::= expr LIKE expr */ - { 264, -5 }, /* (221) expr ::= expr IN LP exprlist RP */ - { 273, -3 }, /* (222) exprlist ::= exprlist COMMA expritem */ - { 273, -1 }, /* (223) exprlist ::= expritem */ - { 274, -1 }, /* (224) expritem ::= expr */ - { 274, 0 }, /* (225) expritem ::= */ - { 209, -3 }, /* (226) cmd ::= RESET QUERY CACHE */ - { 209, -7 }, /* (227) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 209, -7 }, /* (228) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 209, -7 }, /* (229) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 209, -7 }, /* (230) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 209, -8 }, /* (231) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 209, -9 }, /* (232) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 209, -3 }, /* (233) cmd ::= KILL CONNECTION INTEGER */ - { 209, -5 }, /* (234) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 209, -5 }, /* (235) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 209, -1 }, /* (0) program ::= cmd */ + { 210, -2 }, /* (1) cmd ::= SHOW DATABASES */ + { 210, -2 }, /* (2) cmd ::= SHOW MNODES */ + { 210, -2 }, /* (3) cmd ::= SHOW DNODES */ + { 210, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ + { 210, -2 }, /* (5) cmd ::= SHOW USERS */ + { 210, -2 }, /* (6) cmd ::= SHOW MODULES */ + { 210, -2 }, /* (7) cmd ::= SHOW QUERIES */ + { 210, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ + { 210, -2 }, /* (9) cmd ::= SHOW STREAMS */ + { 210, -2 }, /* (10) cmd ::= SHOW VARIABLES */ + { 210, -2 }, /* (11) cmd ::= SHOW SCORES */ + { 210, -2 }, /* (12) cmd ::= SHOW GRANTS */ + { 210, -2 }, /* (13) cmd ::= SHOW VNODES */ + { 210, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ + { 211, 0 }, /* (15) dbPrefix ::= */ + { 211, -2 }, /* (16) dbPrefix ::= ids DOT */ + { 213, 0 }, /* (17) cpxName ::= */ + { 213, -2 }, /* (18) cpxName ::= DOT ids */ + { 210, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ + { 210, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ + { 210, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ + { 210, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + { 210, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ + { 210, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + { 210, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ + { 210, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ + { 210, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ + { 210, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ + { 210, -3 }, /* (29) cmd ::= DROP DNODE ids */ + { 210, -3 }, /* (30) cmd ::= DROP USER ids */ + { 210, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ + { 210, -2 }, /* (32) cmd ::= USE ids */ + { 210, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ + { 210, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ + { 210, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ + { 210, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ + { 210, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ + { 210, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ + { 210, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ + { 210, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ + { 210, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ + { 210, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + { 212, -1 }, /* (43) ids ::= ID */ + { 212, -1 }, /* (44) ids ::= STRING */ + { 214, -2 }, /* (45) ifexists ::= IF EXISTS */ + { 214, 0 }, /* (46) ifexists ::= */ + { 217, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ + { 217, 0 }, /* (48) ifnotexists ::= */ + { 210, -3 }, /* (49) cmd ::= CREATE DNODE ids */ + { 210, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + { 210, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + { 210, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ + { 219, 0 }, /* (53) pps ::= */ + { 219, -2 }, /* (54) pps ::= PPS INTEGER */ + { 220, 0 }, /* (55) tseries ::= */ + { 220, -2 }, /* (56) tseries ::= TSERIES INTEGER */ + { 221, 0 }, /* (57) dbs ::= */ + { 221, -2 }, /* (58) dbs ::= DBS INTEGER */ + { 222, 0 }, /* (59) streams ::= */ + { 222, -2 }, /* (60) streams ::= STREAMS INTEGER */ + { 223, 0 }, /* (61) storage ::= */ + { 223, -2 }, /* (62) storage ::= STORAGE INTEGER */ + { 224, 0 }, /* (63) qtime ::= */ + { 224, -2 }, /* (64) qtime ::= QTIME INTEGER */ + { 225, 0 }, /* (65) users ::= */ + { 225, -2 }, /* (66) users ::= USERS INTEGER */ + { 226, 0 }, /* (67) conns ::= */ + { 226, -2 }, /* (68) conns ::= CONNS INTEGER */ + { 227, 0 }, /* (69) state ::= */ + { 227, -2 }, /* (70) state ::= STATE ids */ + { 216, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + { 228, -2 }, /* (72) keep ::= KEEP tagitemlist */ + { 230, -2 }, /* (73) cache ::= CACHE INTEGER */ + { 231, -2 }, /* (74) replica ::= REPLICA INTEGER */ + { 232, -2 }, /* (75) quorum ::= QUORUM INTEGER */ + { 233, -2 }, /* (76) days ::= DAYS INTEGER */ + { 234, -2 }, /* (77) minrows ::= MINROWS INTEGER */ + { 235, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ + { 236, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ + { 237, -2 }, /* (80) ctime ::= CTIME INTEGER */ + { 238, -2 }, /* (81) wal ::= WAL INTEGER */ + { 239, -2 }, /* (82) fsync ::= FSYNC INTEGER */ + { 240, -2 }, /* (83) comp ::= COMP INTEGER */ + { 241, -2 }, /* (84) prec ::= PRECISION STRING */ + { 242, -2 }, /* (85) update ::= UPDATE INTEGER */ + { 243, -2 }, /* (86) cachelast ::= CACHELAST INTEGER */ + { 218, 0 }, /* (87) db_optr ::= */ + { 218, -2 }, /* (88) db_optr ::= db_optr cache */ + { 218, -2 }, /* (89) db_optr ::= db_optr replica */ + { 218, -2 }, /* (90) db_optr ::= db_optr quorum */ + { 218, -2 }, /* (91) db_optr ::= db_optr days */ + { 218, -2 }, /* (92) db_optr ::= db_optr minrows */ + { 218, -2 }, /* (93) db_optr ::= db_optr maxrows */ + { 218, -2 }, /* (94) db_optr ::= db_optr blocks */ + { 218, -2 }, /* (95) db_optr ::= db_optr ctime */ + { 218, -2 }, /* (96) db_optr ::= db_optr wal */ + { 218, -2 }, /* (97) db_optr ::= db_optr fsync */ + { 218, -2 }, /* (98) db_optr ::= db_optr comp */ + { 218, -2 }, /* (99) db_optr ::= db_optr prec */ + { 218, -2 }, /* (100) db_optr ::= db_optr keep */ + { 218, -2 }, /* (101) db_optr ::= db_optr update */ + { 218, -2 }, /* (102) db_optr ::= db_optr cachelast */ + { 215, 0 }, /* (103) alter_db_optr ::= */ + { 215, -2 }, /* (104) alter_db_optr ::= alter_db_optr replica */ + { 215, -2 }, /* (105) alter_db_optr ::= alter_db_optr quorum */ + { 215, -2 }, /* (106) alter_db_optr ::= alter_db_optr keep */ + { 215, -2 }, /* (107) alter_db_optr ::= alter_db_optr blocks */ + { 215, -2 }, /* (108) alter_db_optr ::= alter_db_optr comp */ + { 215, -2 }, /* (109) alter_db_optr ::= alter_db_optr wal */ + { 215, -2 }, /* (110) alter_db_optr ::= alter_db_optr fsync */ + { 215, -2 }, /* (111) alter_db_optr ::= alter_db_optr update */ + { 215, -2 }, /* (112) alter_db_optr ::= alter_db_optr cachelast */ + { 244, -1 }, /* (113) typename ::= ids */ + { 244, -4 }, /* (114) typename ::= ids LP signed RP */ + { 245, -1 }, /* (115) signed ::= INTEGER */ + { 245, -2 }, /* (116) signed ::= PLUS INTEGER */ + { 245, -2 }, /* (117) signed ::= MINUS INTEGER */ + { 210, -3 }, /* (118) cmd ::= CREATE TABLE create_table_args */ + { 210, -3 }, /* (119) cmd ::= CREATE TABLE create_table_list */ + { 247, -1 }, /* (120) create_table_list ::= create_from_stable */ + { 247, -2 }, /* (121) create_table_list ::= create_table_list create_from_stable */ + { 246, -6 }, /* (122) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + { 246, -10 }, /* (123) create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + { 248, -10 }, /* (124) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + { 246, -5 }, /* (125) create_table_args ::= ifnotexists ids cpxName AS select */ + { 249, -3 }, /* (126) columnlist ::= columnlist COMMA column */ + { 249, -1 }, /* (127) columnlist ::= column */ + { 251, -2 }, /* (128) column ::= ids typename */ + { 229, -3 }, /* (129) tagitemlist ::= tagitemlist COMMA tagitem */ + { 229, -1 }, /* (130) tagitemlist ::= tagitem */ + { 252, -1 }, /* (131) tagitem ::= INTEGER */ + { 252, -1 }, /* (132) tagitem ::= FLOAT */ + { 252, -1 }, /* (133) tagitem ::= STRING */ + { 252, -1 }, /* (134) tagitem ::= BOOL */ + { 252, -1 }, /* (135) tagitem ::= NULL */ + { 252, -2 }, /* (136) tagitem ::= MINUS INTEGER */ + { 252, -2 }, /* (137) tagitem ::= MINUS FLOAT */ + { 252, -2 }, /* (138) tagitem ::= PLUS INTEGER */ + { 252, -2 }, /* (139) tagitem ::= PLUS FLOAT */ + { 250, -12 }, /* (140) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + { 264, -1 }, /* (141) union ::= select */ + { 264, -3 }, /* (142) union ::= LP union RP */ + { 264, -4 }, /* (143) union ::= union UNION ALL select */ + { 264, -6 }, /* (144) union ::= union UNION ALL LP select RP */ + { 210, -1 }, /* (145) cmd ::= union */ + { 250, -2 }, /* (146) select ::= SELECT selcollist */ + { 265, -2 }, /* (147) sclp ::= selcollist COMMA */ + { 265, 0 }, /* (148) sclp ::= */ + { 253, -3 }, /* (149) selcollist ::= sclp expr as */ + { 253, -2 }, /* (150) selcollist ::= sclp STAR */ + { 267, -2 }, /* (151) as ::= AS ids */ + { 267, -1 }, /* (152) as ::= ids */ + { 267, 0 }, /* (153) as ::= */ + { 254, -2 }, /* (154) from ::= FROM tablelist */ + { 268, -2 }, /* (155) tablelist ::= ids cpxName */ + { 268, -3 }, /* (156) tablelist ::= ids cpxName ids */ + { 268, -4 }, /* (157) tablelist ::= tablelist COMMA ids cpxName */ + { 268, -5 }, /* (158) tablelist ::= tablelist COMMA ids cpxName ids */ + { 269, -1 }, /* (159) tmvar ::= VARIABLE */ + { 256, -4 }, /* (160) interval_opt ::= INTERVAL LP tmvar RP */ + { 256, -6 }, /* (161) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + { 256, 0 }, /* (162) interval_opt ::= */ + { 257, 0 }, /* (163) fill_opt ::= */ + { 257, -6 }, /* (164) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 257, -4 }, /* (165) fill_opt ::= FILL LP ID RP */ + { 258, -4 }, /* (166) sliding_opt ::= SLIDING LP tmvar RP */ + { 258, 0 }, /* (167) sliding_opt ::= */ + { 260, 0 }, /* (168) orderby_opt ::= */ + { 260, -3 }, /* (169) orderby_opt ::= ORDER BY sortlist */ + { 270, -4 }, /* (170) sortlist ::= sortlist COMMA item sortorder */ + { 270, -2 }, /* (171) sortlist ::= item sortorder */ + { 272, -2 }, /* (172) item ::= ids cpxName */ + { 273, -1 }, /* (173) sortorder ::= ASC */ + { 273, -1 }, /* (174) sortorder ::= DESC */ + { 273, 0 }, /* (175) sortorder ::= */ + { 259, 0 }, /* (176) groupby_opt ::= */ + { 259, -3 }, /* (177) groupby_opt ::= GROUP BY grouplist */ + { 274, -3 }, /* (178) grouplist ::= grouplist COMMA item */ + { 274, -1 }, /* (179) grouplist ::= item */ + { 261, 0 }, /* (180) having_opt ::= */ + { 261, -2 }, /* (181) having_opt ::= HAVING expr */ + { 263, 0 }, /* (182) limit_opt ::= */ + { 263, -2 }, /* (183) limit_opt ::= LIMIT signed */ + { 263, -4 }, /* (184) limit_opt ::= LIMIT signed OFFSET signed */ + { 263, -4 }, /* (185) limit_opt ::= LIMIT signed COMMA signed */ + { 262, 0 }, /* (186) slimit_opt ::= */ + { 262, -2 }, /* (187) slimit_opt ::= SLIMIT signed */ + { 262, -4 }, /* (188) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 262, -4 }, /* (189) slimit_opt ::= SLIMIT signed COMMA signed */ + { 255, 0 }, /* (190) where_opt ::= */ + { 255, -2 }, /* (191) where_opt ::= WHERE expr */ + { 266, -3 }, /* (192) expr ::= LP expr RP */ + { 266, -1 }, /* (193) expr ::= ID */ + { 266, -3 }, /* (194) expr ::= ID DOT ID */ + { 266, -3 }, /* (195) expr ::= ID DOT STAR */ + { 266, -1 }, /* (196) expr ::= INTEGER */ + { 266, -2 }, /* (197) expr ::= MINUS INTEGER */ + { 266, -2 }, /* (198) expr ::= PLUS INTEGER */ + { 266, -1 }, /* (199) expr ::= FLOAT */ + { 266, -2 }, /* (200) expr ::= MINUS FLOAT */ + { 266, -2 }, /* (201) expr ::= PLUS FLOAT */ + { 266, -1 }, /* (202) expr ::= STRING */ + { 266, -1 }, /* (203) expr ::= NOW */ + { 266, -1 }, /* (204) expr ::= VARIABLE */ + { 266, -1 }, /* (205) expr ::= BOOL */ + { 266, -4 }, /* (206) expr ::= ID LP exprlist RP */ + { 266, -4 }, /* (207) expr ::= ID LP STAR RP */ + { 266, -3 }, /* (208) expr ::= expr IS NULL */ + { 266, -4 }, /* (209) expr ::= expr IS NOT NULL */ + { 266, -3 }, /* (210) expr ::= expr LT expr */ + { 266, -3 }, /* (211) expr ::= expr GT expr */ + { 266, -3 }, /* (212) expr ::= expr LE expr */ + { 266, -3 }, /* (213) expr ::= expr GE expr */ + { 266, -3 }, /* (214) expr ::= expr NE expr */ + { 266, -3 }, /* (215) expr ::= expr EQ expr */ + { 266, -3 }, /* (216) expr ::= expr AND expr */ + { 266, -3 }, /* (217) expr ::= expr OR expr */ + { 266, -3 }, /* (218) expr ::= expr PLUS expr */ + { 266, -3 }, /* (219) expr ::= expr MINUS expr */ + { 266, -3 }, /* (220) expr ::= expr STAR expr */ + { 266, -3 }, /* (221) expr ::= expr SLASH expr */ + { 266, -3 }, /* (222) expr ::= expr REM expr */ + { 266, -3 }, /* (223) expr ::= expr LIKE expr */ + { 266, -5 }, /* (224) expr ::= expr IN LP exprlist RP */ + { 275, -3 }, /* (225) exprlist ::= exprlist COMMA expritem */ + { 275, -1 }, /* (226) exprlist ::= expritem */ + { 276, -1 }, /* (227) expritem ::= expr */ + { 276, 0 }, /* (228) expritem ::= */ + { 210, -3 }, /* (229) cmd ::= RESET QUERY CACHE */ + { 210, -7 }, /* (230) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 210, -7 }, /* (231) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 210, -7 }, /* (232) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 210, -7 }, /* (233) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 210, -8 }, /* (234) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 210, -9 }, /* (235) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 210, -3 }, /* (236) cmd ::= KILL CONNECTION INTEGER */ + { 210, -5 }, /* (237) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 210, -5 }, /* (238) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2027,7 +2038,7 @@ static void yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* program ::= cmd */ - case 115: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==115); + case 118: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==118); {} break; case 1: /* cmd ::= SHOW DATABASES */ @@ -2139,8 +2150,7 @@ static void yy_reduce( } break; case 28: /* cmd ::= DROP DATABASE ifexists ids */ -{ - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0); } +{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0); } break; case 29: /* cmd ::= DROP DNODE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } @@ -2161,12 +2171,10 @@ static void yy_reduce( } break; case 34: /* cmd ::= ALTER USER ids PASS ids */ -{ - setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } +{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; case 35: /* cmd ::= ALTER USER ids PRIVILEGE ids */ -{ - setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} +{ setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} break; case 36: /* cmd ::= ALTER DNODE ids ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } @@ -2181,15 +2189,13 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 40: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy148, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy478, &t);} break; case 41: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ - setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy309);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy463);} break; case 42: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ - setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy309);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy463);} break; case 43: /* ids ::= ID */ case 44: /* ids ::= STRING */ yytestcase(yyruleno==44); @@ -2210,16 +2216,13 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 50: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ - setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, - &yymsp[0].minor.yy309);} +{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy463);} break; case 51: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy148, &yymsp[-2].minor.yy0);} +{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy478, &yymsp[-2].minor.yy0);} break; case 52: /* cmd ::= CREATE USER ids PASS ids */ -{ - setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} +{ setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} break; case 53: /* pps ::= */ case 55: /* tseries ::= */ yytestcase(yyruleno==55); @@ -2245,20 +2248,20 @@ static void yy_reduce( break; case 71: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy309.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy309.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy309.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy309.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy309.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy309.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy309.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy309.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy309.stat = yymsp[0].minor.yy0; -} - yymsp[-8].minor.yy309 = yylhsminor.yy309; + yylhsminor.yy463.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy463.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy463.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy463.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy463.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy463.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy463.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy463.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy463.stat = yymsp[0].minor.yy0; +} + yymsp[-8].minor.yy463 = yylhsminor.yy463; break; case 72: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy131 = yymsp[0].minor.yy131; } +{ yymsp[-1].minor.yy403 = yymsp[0].minor.yy403; } break; case 73: /* cache ::= CACHE INTEGER */ case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74); @@ -2273,591 +2276,588 @@ static void yy_reduce( case 83: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==83); case 84: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==84); case 85: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==85); + case 86: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==86); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 86: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy148);} - break; - case 87: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 88: /* db_optr ::= db_optr replica */ - case 102: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==102); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 89: /* db_optr ::= db_optr quorum */ - case 103: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==103); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 90: /* db_optr ::= db_optr days */ -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 91: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 92: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 93: /* db_optr ::= db_optr blocks */ - case 105: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==105); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 94: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 95: /* db_optr ::= db_optr wal */ - case 107: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==107); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 96: /* db_optr ::= db_optr fsync */ - case 108: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==108); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 97: /* db_optr ::= db_optr comp */ - case 106: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==106); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 98: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 99: /* db_optr ::= db_optr keep */ - case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.keep = yymsp[0].minor.yy131; } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 100: /* db_optr ::= db_optr update */ - case 109: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==109); -{ yylhsminor.yy148 = yymsp[-1].minor.yy148; yylhsminor.yy148.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; - break; - case 101: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy148);} - break; - case 110: /* typename ::= ids */ + case 87: /* db_optr ::= */ +{setDefaultCreateDbOption(&yymsp[1].minor.yy478);} + break; + case 88: /* db_optr ::= db_optr cache */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 89: /* db_optr ::= db_optr replica */ + case 104: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==104); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 90: /* db_optr ::= db_optr quorum */ + case 105: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==105); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 91: /* db_optr ::= db_optr days */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 92: /* db_optr ::= db_optr minrows */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 93: /* db_optr ::= db_optr maxrows */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 94: /* db_optr ::= db_optr blocks */ + case 107: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==107); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 95: /* db_optr ::= db_optr ctime */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 96: /* db_optr ::= db_optr wal */ + case 109: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==109); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 97: /* db_optr ::= db_optr fsync */ + case 110: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==110); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 98: /* db_optr ::= db_optr comp */ + case 108: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==108); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 99: /* db_optr ::= db_optr prec */ +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 100: /* db_optr ::= db_optr keep */ + case 106: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==106); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.keep = yymsp[0].minor.yy403; } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 101: /* db_optr ::= db_optr update */ + case 111: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==111); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 102: /* db_optr ::= db_optr cachelast */ + case 112: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==112); +{ yylhsminor.yy478 = yymsp[-1].minor.yy478; yylhsminor.yy478.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy478 = yylhsminor.yy478; + break; + case 103: /* alter_db_optr ::= */ +{ setDefaultCreateDbOption(&yymsp[1].minor.yy478);} + break; + case 113: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSqlSetColumnType(&yylhsminor.yy163, &yymsp[0].minor.yy0); + tSqlSetColumnType (&yylhsminor.yy363, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy163 = yylhsminor.yy163; + yymsp[0].minor.yy363 = yylhsminor.yy363; break; - case 111: /* typename ::= ids LP signed RP */ + case 114: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy459 <= 0) { + if (yymsp[-1].minor.yy387 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSqlSetColumnType(&yylhsminor.yy163, &yymsp[-3].minor.yy0); + tSqlSetColumnType(&yylhsminor.yy363, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy459; // negative value of name length - tSqlSetColumnType(&yylhsminor.yy163, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy387; // negative value of name length + tSqlSetColumnType(&yylhsminor.yy363, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy163 = yylhsminor.yy163; + yymsp[-3].minor.yy363 = yylhsminor.yy363; break; - case 112: /* signed ::= INTEGER */ -{ yylhsminor.yy459 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy459 = yylhsminor.yy459; + case 115: /* signed ::= INTEGER */ +{ yylhsminor.yy387 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy387 = yylhsminor.yy387; break; - case 113: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy459 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 116: /* signed ::= PLUS INTEGER */ +{ yymsp[-1].minor.yy387 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 114: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy459 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} + case 117: /* signed ::= MINUS INTEGER */ +{ yymsp[-1].minor.yy387 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 116: /* cmd ::= CREATE TABLE create_table_list */ -{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy538;} + case 119: /* cmd ::= CREATE TABLE create_table_list */ +{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy436;} break; - case 117: /* create_table_list ::= create_from_stable */ + case 120: /* create_table_list ::= create_from_stable */ { SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy96); + taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy84); pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - yylhsminor.yy538 = pCreateTable; + yylhsminor.yy436 = pCreateTable; } - yymsp[0].minor.yy538 = yylhsminor.yy538; + yymsp[0].minor.yy436 = yylhsminor.yy436; break; - case 118: /* create_table_list ::= create_table_list create_from_stable */ + case 121: /* create_table_list ::= create_table_list create_from_stable */ { - taosArrayPush(yymsp[-1].minor.yy538->childTableInfo, &yymsp[0].minor.yy96); - yylhsminor.yy538 = yymsp[-1].minor.yy538; + taosArrayPush(yymsp[-1].minor.yy436->childTableInfo, &yymsp[0].minor.yy84); + yylhsminor.yy436 = yymsp[-1].minor.yy436; } - yymsp[-1].minor.yy538 = yylhsminor.yy538; + yymsp[-1].minor.yy436 = yylhsminor.yy436; break; - case 119: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + case 122: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { - yylhsminor.yy538 = tSetCreateSqlElems(yymsp[-1].minor.yy131, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, yylhsminor.yy538, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy436 = tSetCreateSqlElems(yymsp[-1].minor.yy403, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, yylhsminor.yy436, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy538 = yylhsminor.yy538; + yymsp[-5].minor.yy436 = yylhsminor.yy436; break; - case 120: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + case 123: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { - yylhsminor.yy538 = tSetCreateSqlElems(yymsp[-5].minor.yy131, yymsp[-1].minor.yy131, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, yylhsminor.yy538, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy436 = tSetCreateSqlElems(yymsp[-5].minor.yy403, yymsp[-1].minor.yy403, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, yylhsminor.yy436, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy538 = yylhsminor.yy538; + yymsp[-9].minor.yy436 = yylhsminor.yy436; break; - case 121: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + case 124: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yylhsminor.yy96 = createNewChildTableInfo(&yymsp[-5].minor.yy0, yymsp[-1].minor.yy131, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); + yylhsminor.yy84 = createNewChildTableInfo(&yymsp[-5].minor.yy0, yymsp[-1].minor.yy403, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy96 = yylhsminor.yy96; + yymsp[-9].minor.yy84 = yylhsminor.yy84; break; - case 122: /* create_table_args ::= ifnotexists ids cpxName AS select */ + case 125: /* create_table_args ::= ifnotexists ids cpxName AS select */ { - yylhsminor.yy538 = tSetCreateSqlElems(NULL, NULL, yymsp[0].minor.yy84, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, yylhsminor.yy538, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy436 = tSetCreateSqlElems(NULL, NULL, yymsp[0].minor.yy4, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, yylhsminor.yy436, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy538 = yylhsminor.yy538; + yymsp[-4].minor.yy436 = yylhsminor.yy436; break; - case 123: /* columnlist ::= columnlist COMMA column */ -{taosArrayPush(yymsp[-2].minor.yy131, &yymsp[0].minor.yy163); yylhsminor.yy131 = yymsp[-2].minor.yy131; } - yymsp[-2].minor.yy131 = yylhsminor.yy131; + case 126: /* columnlist ::= columnlist COMMA column */ +{taosArrayPush(yymsp[-2].minor.yy403, &yymsp[0].minor.yy363); yylhsminor.yy403 = yymsp[-2].minor.yy403; } + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 124: /* columnlist ::= column */ -{yylhsminor.yy131 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy131, &yymsp[0].minor.yy163);} - yymsp[0].minor.yy131 = yylhsminor.yy131; + case 127: /* columnlist ::= column */ +{yylhsminor.yy403 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy403, &yymsp[0].minor.yy363);} + yymsp[0].minor.yy403 = yylhsminor.yy403; break; - case 125: /* column ::= ids typename */ + case 128: /* column ::= ids typename */ { - tSqlSetColumnInfo(&yylhsminor.yy163, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy163); + tSqlSetColumnInfo(&yylhsminor.yy363, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy363); } - yymsp[-1].minor.yy163 = yylhsminor.yy163; + yymsp[-1].minor.yy363 = yylhsminor.yy363; break; - case 126: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy131 = tVariantListAppend(yymsp[-2].minor.yy131, &yymsp[0].minor.yy516, -1); } - yymsp[-2].minor.yy131 = yylhsminor.yy131; + case 129: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yylhsminor.yy403 = tVariantListAppend(yymsp[-2].minor.yy403, &yymsp[0].minor.yy488, -1); } + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 127: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy131 = tVariantListAppend(NULL, &yymsp[0].minor.yy516, -1); } - yymsp[0].minor.yy131 = yylhsminor.yy131; + case 130: /* tagitemlist ::= tagitem */ +{ yylhsminor.yy403 = tVariantListAppend(NULL, &yymsp[0].minor.yy488, -1); } + yymsp[0].minor.yy403 = yylhsminor.yy403; break; - case 128: /* tagitem ::= INTEGER */ - case 129: /* tagitem ::= FLOAT */ yytestcase(yyruleno==129); - case 130: /* tagitem ::= STRING */ yytestcase(yyruleno==130); - case 131: /* tagitem ::= BOOL */ yytestcase(yyruleno==131); -{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy516, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy516 = yylhsminor.yy516; + case 131: /* tagitem ::= INTEGER */ + case 132: /* tagitem ::= FLOAT */ yytestcase(yyruleno==132); + case 133: /* tagitem ::= STRING */ yytestcase(yyruleno==133); + case 134: /* tagitem ::= BOOL */ yytestcase(yyruleno==134); +{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy488, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy488 = yylhsminor.yy488; break; - case 132: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy516, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy516 = yylhsminor.yy516; + case 135: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy488, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy488 = yylhsminor.yy488; break; - case 133: /* tagitem ::= MINUS INTEGER */ - case 134: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==134); - case 135: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==135); - case 136: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==136); + case 136: /* tagitem ::= MINUS INTEGER */ + case 137: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==137); + case 138: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==138); + case 139: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==139); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy516, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy488, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy516 = yylhsminor.yy516; + yymsp[-1].minor.yy488 = yylhsminor.yy488; break; - case 137: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 140: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy84 = tSetQuerySqlElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy478, yymsp[-9].minor.yy131, - yymsp[-8].minor.yy420, yymsp[-4].minor.yy131, yymsp[-3].minor.yy131, - &yymsp[-7].minor.yy530, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy131, - &yymsp[0].minor.yy284, &yymsp[-1].minor.yy284); + yylhsminor.yy4 = tSetQuerySqlElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy382, yymsp[-9].minor.yy403, yymsp[-8].minor.yy522, yymsp[-4].minor.yy403, yymsp[-3].minor.yy403, &yymsp[-7].minor.yy222, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy403, &yymsp[0].minor.yy404, &yymsp[-1].minor.yy404); } - yymsp[-11].minor.yy84 = yylhsminor.yy84; + yymsp[-11].minor.yy4 = yylhsminor.yy4; break; - case 138: /* union ::= select */ -{ yylhsminor.yy513 = setSubclause(NULL, yymsp[0].minor.yy84); } - yymsp[0].minor.yy513 = yylhsminor.yy513; + case 141: /* union ::= select */ +{ yylhsminor.yy13 = setSubclause(NULL, yymsp[0].minor.yy4); } + yymsp[0].minor.yy13 = yylhsminor.yy13; break; - case 139: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy513 = yymsp[-1].minor.yy513; } + case 142: /* union ::= LP union RP */ +{ yymsp[-2].minor.yy13 = yymsp[-1].minor.yy13; } break; - case 140: /* union ::= union UNION ALL select */ -{ yylhsminor.yy513 = appendSelectClause(yymsp[-3].minor.yy513, yymsp[0].minor.yy84); } - yymsp[-3].minor.yy513 = yylhsminor.yy513; + case 143: /* union ::= union UNION ALL select */ +{ yylhsminor.yy13 = appendSelectClause(yymsp[-3].minor.yy13, yymsp[0].minor.yy4); } + yymsp[-3].minor.yy13 = yylhsminor.yy13; break; - case 141: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy513 = appendSelectClause(yymsp[-5].minor.yy513, yymsp[-1].minor.yy84); } - yymsp[-5].minor.yy513 = yylhsminor.yy513; + case 144: /* union ::= union UNION ALL LP select RP */ +{ yylhsminor.yy13 = appendSelectClause(yymsp[-5].minor.yy13, yymsp[-1].minor.yy4); } + yymsp[-5].minor.yy13 = yylhsminor.yy13; break; - case 142: /* cmd ::= union */ -{ - setSqlInfo(pInfo, yymsp[0].minor.yy513, NULL, TSDB_SQL_SELECT); } + case 145: /* cmd ::= union */ +{ setSqlInfo(pInfo, yymsp[0].minor.yy13, NULL, TSDB_SQL_SELECT); } break; - case 143: /* select ::= SELECT selcollist */ + case 146: /* select ::= SELECT selcollist */ { - yylhsminor.yy84 = tSetQuerySqlElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy478, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL); + yylhsminor.yy4 = tSetQuerySqlElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy382, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy84 = yylhsminor.yy84; + yymsp[-1].minor.yy4 = yylhsminor.yy4; break; - case 144: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy478 = yymsp[-1].minor.yy478;} - yymsp[-1].minor.yy478 = yylhsminor.yy478; + case 147: /* sclp ::= selcollist COMMA */ +{yylhsminor.yy382 = yymsp[-1].minor.yy382;} + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; - case 145: /* sclp ::= */ -{yymsp[1].minor.yy478 = 0;} + case 148: /* sclp ::= */ +{yymsp[1].minor.yy382 = 0;} break; - case 146: /* selcollist ::= sclp expr as */ + case 149: /* selcollist ::= sclp expr as */ { - yylhsminor.yy478 = tSqlExprListAppend(yymsp[-2].minor.yy478, yymsp[-1].minor.yy420, - yymsp[0].minor.yy0.n ? &yymsp[0].minor.yy0 : 0); + yylhsminor.yy382 = tSqlExprListAppend(yymsp[-2].minor.yy382, yymsp[-1].minor.yy522, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy478 = yylhsminor.yy478; + yymsp[-2].minor.yy382 = yylhsminor.yy382; break; - case 147: /* selcollist ::= sclp STAR */ + case 150: /* selcollist ::= sclp STAR */ { tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy478 = tSqlExprListAppend(yymsp[-1].minor.yy478, pNode, 0); + yylhsminor.yy382 = tSqlExprListAppend(yymsp[-1].minor.yy382, pNode, 0); } - yymsp[-1].minor.yy478 = yylhsminor.yy478; + yymsp[-1].minor.yy382 = yylhsminor.yy382; break; - case 148: /* as ::= AS ids */ + case 151: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 149: /* as ::= ids */ + case 152: /* as ::= ids */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 150: /* as ::= */ + case 153: /* as ::= */ { yymsp[1].minor.yy0.n = 0; } break; - case 151: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy131 = yymsp[0].minor.yy131;} + case 154: /* from ::= FROM tablelist */ +{yymsp[-1].minor.yy403 = yymsp[0].minor.yy403;} break; - case 152: /* tablelist ::= ids cpxName */ + case 155: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy131 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy131 = tVariantListAppendToken(yylhsminor.yy131, &yymsp[-1].minor.yy0, -1); // table alias name + yylhsminor.yy403 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy131 = yylhsminor.yy131; + yymsp[-1].minor.yy403 = yylhsminor.yy403; break; - case 153: /* tablelist ::= ids cpxName ids */ + case 156: /* tablelist ::= ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy131 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy131 = tVariantListAppendToken(yylhsminor.yy131, &yymsp[0].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy131 = yylhsminor.yy131; + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 154: /* tablelist ::= tablelist COMMA ids cpxName */ + case 157: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy131 = tVariantListAppendToken(yymsp[-3].minor.yy131, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy131 = tVariantListAppendToken(yylhsminor.yy131, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yymsp[-3].minor.yy403, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy131 = yylhsminor.yy131; + yymsp[-3].minor.yy403 = yylhsminor.yy403; break; - case 155: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 158: /* tablelist ::= tablelist COMMA ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy131 = tVariantListAppendToken(yymsp[-4].minor.yy131, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy131 = tVariantListAppendToken(yylhsminor.yy131, &yymsp[0].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yymsp[-4].minor.yy403, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy403 = tVariantListAppendToken(yylhsminor.yy403, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy131 = yylhsminor.yy131; + yymsp[-4].minor.yy403 = yylhsminor.yy403; break; - case 156: /* tmvar ::= VARIABLE */ + case 159: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 157: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy530.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy530.offset.n = 0; yymsp[-3].minor.yy530.offset.z = NULL; yymsp[-3].minor.yy530.offset.type = 0;} + case 160: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yymsp[-3].minor.yy222.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy222.offset.n = 0; yymsp[-3].minor.yy222.offset.z = NULL; yymsp[-3].minor.yy222.offset.type = 0;} break; - case 158: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy530.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy530.offset = yymsp[-1].minor.yy0;} + case 161: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yymsp[-5].minor.yy222.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy222.offset = yymsp[-1].minor.yy0;} break; - case 159: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy530, 0, sizeof(yymsp[1].minor.yy530));} + case 162: /* interval_opt ::= */ +{memset(&yymsp[1].minor.yy222, 0, sizeof(yymsp[1].minor.yy222));} break; - case 160: /* fill_opt ::= */ -{yymsp[1].minor.yy131 = 0; } + case 163: /* fill_opt ::= */ +{yymsp[1].minor.yy403 = 0; } break; - case 161: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 164: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy131, &A, -1, 0); - yymsp[-5].minor.yy131 = yymsp[-1].minor.yy131; + tVariantListInsert(yymsp[-1].minor.yy403, &A, -1, 0); + yymsp[-5].minor.yy403 = yymsp[-1].minor.yy403; } break; - case 162: /* fill_opt ::= FILL LP ID RP */ + case 165: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy131 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy403 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 163: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 166: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 164: /* sliding_opt ::= */ + case 167: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 165: /* orderby_opt ::= */ -{yymsp[1].minor.yy131 = 0;} + case 168: /* orderby_opt ::= */ +{yymsp[1].minor.yy403 = 0;} break; - case 166: /* orderby_opt ::= ORDER BY sortlist */ -{yymsp[-2].minor.yy131 = yymsp[0].minor.yy131;} + case 169: /* orderby_opt ::= ORDER BY sortlist */ +{yymsp[-2].minor.yy403 = yymsp[0].minor.yy403;} break; - case 167: /* sortlist ::= sortlist COMMA item sortorder */ + case 170: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy131 = tVariantListAppend(yymsp[-3].minor.yy131, &yymsp[-1].minor.yy516, yymsp[0].minor.yy42); + yylhsminor.yy403 = tVariantListAppend(yymsp[-3].minor.yy403, &yymsp[-1].minor.yy488, yymsp[0].minor.yy70); } - yymsp[-3].minor.yy131 = yylhsminor.yy131; + yymsp[-3].minor.yy403 = yylhsminor.yy403; break; - case 168: /* sortlist ::= item sortorder */ + case 171: /* sortlist ::= item sortorder */ { - yylhsminor.yy131 = tVariantListAppend(NULL, &yymsp[-1].minor.yy516, yymsp[0].minor.yy42); + yylhsminor.yy403 = tVariantListAppend(NULL, &yymsp[-1].minor.yy488, yymsp[0].minor.yy70); } - yymsp[-1].minor.yy131 = yylhsminor.yy131; + yymsp[-1].minor.yy403 = yylhsminor.yy403; break; - case 169: /* item ::= ids cpxName */ + case 172: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy516, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy488, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy516 = yylhsminor.yy516; + yymsp[-1].minor.yy488 = yylhsminor.yy488; break; - case 170: /* sortorder ::= ASC */ -{ yymsp[0].minor.yy42 = TSDB_ORDER_ASC; } + case 173: /* sortorder ::= ASC */ +{ yymsp[0].minor.yy70 = TSDB_ORDER_ASC; } break; - case 171: /* sortorder ::= DESC */ -{ yymsp[0].minor.yy42 = TSDB_ORDER_DESC;} + case 174: /* sortorder ::= DESC */ +{ yymsp[0].minor.yy70 = TSDB_ORDER_DESC;} break; - case 172: /* sortorder ::= */ -{ yymsp[1].minor.yy42 = TSDB_ORDER_ASC; } + case 175: /* sortorder ::= */ +{ yymsp[1].minor.yy70 = TSDB_ORDER_ASC; } break; - case 173: /* groupby_opt ::= */ -{ yymsp[1].minor.yy131 = 0;} + case 176: /* groupby_opt ::= */ +{ yymsp[1].minor.yy403 = 0;} break; - case 174: /* groupby_opt ::= GROUP BY grouplist */ -{ yymsp[-2].minor.yy131 = yymsp[0].minor.yy131;} + case 177: /* groupby_opt ::= GROUP BY grouplist */ +{ yymsp[-2].minor.yy403 = yymsp[0].minor.yy403;} break; - case 175: /* grouplist ::= grouplist COMMA item */ + case 178: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy131 = tVariantListAppend(yymsp[-2].minor.yy131, &yymsp[0].minor.yy516, -1); + yylhsminor.yy403 = tVariantListAppend(yymsp[-2].minor.yy403, &yymsp[0].minor.yy488, -1); } - yymsp[-2].minor.yy131 = yylhsminor.yy131; + yymsp[-2].minor.yy403 = yylhsminor.yy403; break; - case 176: /* grouplist ::= item */ + case 179: /* grouplist ::= item */ { - yylhsminor.yy131 = tVariantListAppend(NULL, &yymsp[0].minor.yy516, -1); + yylhsminor.yy403 = tVariantListAppend(NULL, &yymsp[0].minor.yy488, -1); } - yymsp[0].minor.yy131 = yylhsminor.yy131; + yymsp[0].minor.yy403 = yylhsminor.yy403; break; - case 177: /* having_opt ::= */ - case 187: /* where_opt ::= */ yytestcase(yyruleno==187); - case 225: /* expritem ::= */ yytestcase(yyruleno==225); -{yymsp[1].minor.yy420 = 0;} + case 180: /* having_opt ::= */ + case 190: /* where_opt ::= */ yytestcase(yyruleno==190); + case 228: /* expritem ::= */ yytestcase(yyruleno==228); +{yymsp[1].minor.yy522 = 0;} break; - case 178: /* having_opt ::= HAVING expr */ - case 188: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==188); -{yymsp[-1].minor.yy420 = yymsp[0].minor.yy420;} + case 181: /* having_opt ::= HAVING expr */ + case 191: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==191); +{yymsp[-1].minor.yy522 = yymsp[0].minor.yy522;} break; - case 179: /* limit_opt ::= */ - case 183: /* slimit_opt ::= */ yytestcase(yyruleno==183); -{yymsp[1].minor.yy284.limit = -1; yymsp[1].minor.yy284.offset = 0;} + case 182: /* limit_opt ::= */ + case 186: /* slimit_opt ::= */ yytestcase(yyruleno==186); +{yymsp[1].minor.yy404.limit = -1; yymsp[1].minor.yy404.offset = 0;} break; - case 180: /* limit_opt ::= LIMIT signed */ - case 184: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==184); -{yymsp[-1].minor.yy284.limit = yymsp[0].minor.yy459; yymsp[-1].minor.yy284.offset = 0;} + case 183: /* limit_opt ::= LIMIT signed */ + case 187: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==187); +{yymsp[-1].minor.yy404.limit = yymsp[0].minor.yy387; yymsp[-1].minor.yy404.offset = 0;} break; - case 181: /* limit_opt ::= LIMIT signed OFFSET signed */ -{ yymsp[-3].minor.yy284.limit = yymsp[-2].minor.yy459; yymsp[-3].minor.yy284.offset = yymsp[0].minor.yy459;} + case 184: /* limit_opt ::= LIMIT signed OFFSET signed */ +{ yymsp[-3].minor.yy404.limit = yymsp[-2].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[0].minor.yy387;} break; - case 182: /* limit_opt ::= LIMIT signed COMMA signed */ -{ yymsp[-3].minor.yy284.limit = yymsp[0].minor.yy459; yymsp[-3].minor.yy284.offset = yymsp[-2].minor.yy459;} + case 185: /* limit_opt ::= LIMIT signed COMMA signed */ +{ yymsp[-3].minor.yy404.limit = yymsp[0].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[-2].minor.yy387;} break; - case 185: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -{yymsp[-3].minor.yy284.limit = yymsp[-2].minor.yy459; yymsp[-3].minor.yy284.offset = yymsp[0].minor.yy459;} + case 188: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ +{yymsp[-3].minor.yy404.limit = yymsp[-2].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[0].minor.yy387;} break; - case 186: /* slimit_opt ::= SLIMIT signed COMMA signed */ -{yymsp[-3].minor.yy284.limit = yymsp[0].minor.yy459; yymsp[-3].minor.yy284.offset = yymsp[-2].minor.yy459;} + case 189: /* slimit_opt ::= SLIMIT signed COMMA signed */ +{yymsp[-3].minor.yy404.limit = yymsp[0].minor.yy387; yymsp[-3].minor.yy404.offset = yymsp[-2].minor.yy387;} break; - case 189: /* expr ::= LP expr RP */ -{yylhsminor.yy420 = yymsp[-1].minor.yy420; yylhsminor.yy420->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy420->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 192: /* expr ::= LP expr RP */ +{yylhsminor.yy522 = yymsp[-1].minor.yy522; yylhsminor.yy522->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy522->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 190: /* expr ::= ID */ -{ yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 193: /* expr ::= ID */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 191: /* expr ::= ID DOT ID */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 194: /* expr ::= ID DOT ID */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 192: /* expr ::= ID DOT STAR */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 195: /* expr ::= ID DOT STAR */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 193: /* expr ::= INTEGER */ -{ yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 196: /* expr ::= INTEGER */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 194: /* expr ::= MINUS INTEGER */ - case 195: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==195); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy420 = yylhsminor.yy420; + case 197: /* expr ::= MINUS INTEGER */ + case 198: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==198); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy522 = yylhsminor.yy522; break; - case 196: /* expr ::= FLOAT */ -{ yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 199: /* expr ::= FLOAT */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 197: /* expr ::= MINUS FLOAT */ - case 198: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==198); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy420 = yylhsminor.yy420; + case 200: /* expr ::= MINUS FLOAT */ + case 201: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==201); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy522 = yylhsminor.yy522; break; - case 199: /* expr ::= STRING */ -{ yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 202: /* expr ::= STRING */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 200: /* expr ::= NOW */ -{ yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 203: /* expr ::= NOW */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 201: /* expr ::= VARIABLE */ -{ yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 204: /* expr ::= VARIABLE */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 202: /* expr ::= BOOL */ -{ yylhsminor.yy420 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 205: /* expr ::= BOOL */ +{ yylhsminor.yy522 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 203: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy420 = tSqlExprCreateFunction(yymsp[-1].minor.yy478, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, - yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy420 = yylhsminor.yy420; + case 206: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy522 = tSqlExprCreateFunction(yymsp[-1].minor.yy382, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy522 = yylhsminor.yy522; break; - case 204: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy420 = - tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy420 = yylhsminor.yy420; + case 207: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy522 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy522 = yylhsminor.yy522; break; - case 205: /* expr ::= expr IS NULL */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, NULL, TK_ISNULL);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 208: /* expr ::= expr IS NULL */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, NULL, TK_ISNULL);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 206: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-3].minor.yy420, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy420 = yylhsminor.yy420; + case 209: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-3].minor.yy522, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy522 = yylhsminor.yy522; break; - case 207: /* expr ::= expr LT expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_LT);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 210: /* expr ::= expr LT expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_LT);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 208: /* expr ::= expr GT expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_GT);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 211: /* expr ::= expr GT expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_GT);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 209: /* expr ::= expr LE expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_LE);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 212: /* expr ::= expr LE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_LE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 210: /* expr ::= expr GE expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_GE);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 213: /* expr ::= expr GE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_GE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 211: /* expr ::= expr NE expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_NE);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 214: /* expr ::= expr NE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_NE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 212: /* expr ::= expr EQ expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_EQ);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 215: /* expr ::= expr EQ expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_EQ);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 213: /* expr ::= expr AND expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_AND);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 216: /* expr ::= expr AND expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_AND);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 214: /* expr ::= expr OR expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_OR); } - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 217: /* expr ::= expr OR expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_OR); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 215: /* expr ::= expr PLUS expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_PLUS); } - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 218: /* expr ::= expr PLUS expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_PLUS); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 216: /* expr ::= expr MINUS expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_MINUS); } - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 219: /* expr ::= expr MINUS expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_MINUS); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 217: /* expr ::= expr STAR expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_STAR); } - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 220: /* expr ::= expr STAR expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_STAR); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 218: /* expr ::= expr SLASH expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_DIVIDE);} - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 221: /* expr ::= expr SLASH expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_DIVIDE);} + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 219: /* expr ::= expr REM expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_REM); } - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 222: /* expr ::= expr REM expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_REM); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 220: /* expr ::= expr LIKE expr */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-2].minor.yy420, yymsp[0].minor.yy420, TK_LIKE); } - yymsp[-2].minor.yy420 = yylhsminor.yy420; + case 223: /* expr ::= expr LIKE expr */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-2].minor.yy522, yymsp[0].minor.yy522, TK_LIKE); } + yymsp[-2].minor.yy522 = yylhsminor.yy522; break; - case 221: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy420 = tSqlExprCreate(yymsp[-4].minor.yy420, (tSQLExpr *)yymsp[-1].minor.yy478, TK_IN); } - yymsp[-4].minor.yy420 = yylhsminor.yy420; + case 224: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy522 = tSqlExprCreate(yymsp[-4].minor.yy522, (tSQLExpr*)yymsp[-1].minor.yy382, TK_IN); } + yymsp[-4].minor.yy522 = yylhsminor.yy522; break; - case 222: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy478 = tSqlExprListAppend(yymsp[-2].minor.yy478, yymsp[0].minor.yy420, 0);} - yymsp[-2].minor.yy478 = yylhsminor.yy478; + case 225: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy382 = tSqlExprListAppend(yymsp[-2].minor.yy382,yymsp[0].minor.yy522,0);} + yymsp[-2].minor.yy382 = yylhsminor.yy382; break; - case 223: /* exprlist ::= expritem */ -{yylhsminor.yy478 = tSqlExprListAppend(0, yymsp[0].minor.yy420, 0);} - yymsp[0].minor.yy478 = yylhsminor.yy478; + case 226: /* exprlist ::= expritem */ +{yylhsminor.yy382 = tSqlExprListAppend(0,yymsp[0].minor.yy522,0);} + yymsp[0].minor.yy382 = yylhsminor.yy382; break; - case 224: /* expritem ::= expr */ -{yylhsminor.yy420 = yymsp[0].minor.yy420;} - yymsp[0].minor.yy420 = yylhsminor.yy420; + case 227: /* expritem ::= expr */ +{yylhsminor.yy522 = yymsp[0].minor.yy522;} + yymsp[0].minor.yy522 = yylhsminor.yy522; break; - case 226: /* cmd ::= RESET QUERY CACHE */ + case 229: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 227: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 230: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = - tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy403, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 228: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 231: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2868,15 +2868,14 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 229: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 232: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = - tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy403, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 230: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 233: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2887,7 +2886,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 231: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 234: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2897,34 +2896,30 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = - tAlterTableSqlElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 232: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 235: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy516, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy488, -1); SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 233: /* cmd ::= KILL CONNECTION INTEGER */ -{ - setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} + case 236: /* cmd ::= KILL CONNECTION INTEGER */ +{setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 234: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); - setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} + case 237: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ +{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 235: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ -{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); - setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} + case 238: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ +{yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: break; diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index 4a5f7fbbbed18b0a2c2cd7fff7ad3101cfa11c41..82fd4bcf9907eeb1286f9b6d0976e541e7b1a16b 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -21,19 +21,19 @@ TEST(testCase, histogram_binary_search) { pHisto->elems[i].val = i; } - int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1); + int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1); assert(idx == 1); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9); assert(idx == 9); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20); assert(idx == 10); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1); assert(idx == 0); - idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); + idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9); assert(idx == 4); free(pHisto); diff --git a/src/rpc/inc/rpcHead.h b/src/rpc/inc/rpcHead.h index 520edadc7dd072849720cad53c7f6f4ba605a06c..5b401ac54be194a38bad2f7680155f5edb8350b4 100644 --- a/src/rpc/inc/rpcHead.h +++ b/src/rpc/inc/rpcHead.h @@ -20,10 +20,6 @@ extern "C" { #endif -#define RPC_CONN_UDPS 0 -#define RPC_CONN_UDPC 1 -#define RPC_CONN_TCPS 2 -#define RPC_CONN_TCPC 3 #define RPC_CONN_TCP 2 extern int tsRpcOverhead; @@ -58,6 +54,7 @@ typedef struct { char empty[1]; // reserved uint8_t msgType; // message type int32_t msgLen; // message length including the header iteslf + uint32_t msgVer; int32_t code; // code in response message uint8_t content[0]; // message body starts from here } SRpcHead; diff --git a/src/rpc/inc/rpcLog.h b/src/rpc/inc/rpcLog.h index 3c1614cda225f503e511e15acda71a278e2408ba..6c4a281d2c0f0bf85975200c833acbac856dc40c 100644 --- a/src/rpc/inc/rpcLog.h +++ b/src/rpc/inc/rpcLog.h @@ -23,7 +23,7 @@ extern "C" { #include "tlog.h" extern int32_t rpcDebugFlag; -extern uint32_t tscEmbedded; +extern int8_t tscEmbedded; #define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} #define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("RPC ERROR ", tscEmbedded ? 255 : rpcDebugFlag, __VA_ARGS__); }} diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 09d8f3bff1faad5596d85e931adea7f83670a48a..60a12c26b78626ed81cbc182d76c836c6ee74498 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -272,7 +272,7 @@ static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) } static void rpcLockCache(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { @@ -282,7 +282,7 @@ static void rpcLockCache(int64_t *lockedBy) { } static void rpcUnlockCache(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { assert(false); } diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a0c164955656c116f97cd44247cb6381fb444a7d..af51733c21fa417c9a8de6d53e15ec92555f35cd 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -142,7 +142,6 @@ static int32_t tsRpcNum = 0; #define RPC_CONN_UDPC 1 #define RPC_CONN_TCPS 2 #define RPC_CONN_TCPC 3 -#define RPC_CONN_TCP 2 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { taosInitUdpConnection, @@ -404,7 +403,7 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64 if (type == TSDB_MSG_TYPE_QUERY || type == TSDB_MSG_TYPE_CM_RETRIEVE || type == TSDB_MSG_TYPE_FETCH || type == TSDB_MSG_TYPE_CM_STABLE_VGROUP || type == TSDB_MSG_TYPE_CM_TABLES_META || type == TSDB_MSG_TYPE_CM_TABLE_META - || type == TSDB_MSG_TYPE_CM_SHOW ) + || type == TSDB_MSG_TYPE_CM_SHOW || type == TSDB_MSG_TYPE_DM_STATUS) pContext->connType = RPC_CONN_TCPC; pContext->rid = taosAddRef(tsRpcRefId, pContext); @@ -576,7 +575,7 @@ static void rpcFreeMsg(void *msg) { static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { SRpcConn *pConn; - uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); + uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -959,6 +958,11 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL; } + if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { + tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, taosMsg[pHead->msgType]); + terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL; + } + pConn = rpcGetConnObj(pRpc, sid, pRecv); if (pConn == NULL) { tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); @@ -1212,6 +1216,7 @@ static void rpcSendReqHead(SRpcConn *pConn) { pHead = (SRpcHead *)msg; pHead->version = 1; pHead->msgType = pConn->outType; + pHead->msgVer = htonl(tsVersion >> 8); pHead->spi = pConn->spi; pHead->encrypt = 0; pHead->tranId = pConn->outTranId; @@ -1282,6 +1287,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { // set the message header pHead->version = 1; + pHead->msgVer = htonl(tsVersion >> 8); pHead->msgType = msgType; pHead->encrypt = 0; pConn->tranId++; @@ -1561,6 +1567,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_INVALID_USER || code == TSDB_CODE_RPC_NOT_READY) { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); @@ -1598,7 +1605,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { } static void rpcLockConn(SRpcConn *pConn) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(&(pConn->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { @@ -1608,7 +1615,7 @@ static void rpcLockConn(SRpcConn *pConn) { } static void rpcUnlockConn(SRpcConn *pConn) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(pConn->lockedBy), tid, 0) != tid) { assert(false); } diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index d855c651f93a58d46e064e67d287f135684f4954..47090cfa0cc6997a20c6803b8fc299f08e407bb5 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -38,7 +38,7 @@ extern "C" { #define SYNC_MAX_FWDS 512 #define SYNC_FWD_TIMER 300 #define SYNC_ROLE_TIMER 15000 // ms -#define SYNC_CHECK_INTERVAL 1 // ms +#define SYNC_CHECK_INTERVAL 1000 // ms #define SYNC_WAIT_AFTER_CHOOSE_MASTER 10 // ms #define nodeRole pNode->peerInfo[pNode->selfIndex]->role @@ -86,9 +86,10 @@ typedef struct SsyncPeer { int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process + int32_t refCount; + int64_t rid; void * timer; void * pConn; - int32_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -98,6 +99,7 @@ typedef struct SSyncNode { int8_t quorum; int8_t selfIndex; uint32_t vgId; + int32_t refCount; int64_t rid; SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator SSyncPeer * pMaster; @@ -121,13 +123,13 @@ extern int32_t tsSyncNum; extern char tsNodeFqdn[TSDB_FQDN_LEN]; extern char * syncStatus[]; -void *syncRetrieveData(void *param); -void *syncRestoreData(void *param); -int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); -void syncRestartConnection(SSyncPeer *pPeer); -void syncBroadcastStatus(SSyncNode *pNode); -void syncAddPeerRef(SSyncPeer *pPeer); -int32_t syncDecPeerRef(SSyncPeer *pPeer); +void * syncRetrieveData(void *param); +void * syncRestoreData(void *param); +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); +void syncRestartConnection(SSyncPeer *pPeer); +void syncBroadcastStatus(SSyncNode *pNode); +SSyncPeer *syncAcquirePeer(int64_t rid); +void syncReleasePeer(SSyncPeer *pPeer); #ifdef __cplusplus } diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index 7db51f2a7115ccf23ee42d17cbe62f0798adf260..d4674fee6bd07d5ab68284399d9a6502dbd5ec2c 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -25,14 +25,14 @@ typedef struct { uint32_t serverIp; int16_t port; int32_t bufferSize; - void (*processBrokenLink)(void *ahandle); - int32_t (*processIncomingMsg)(void *ahandle, void *buffer); + void (*processBrokenLink)(int64_t handleId); + int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); void (*processIncomingConn)(int32_t fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, void *ahandle, int32_t connFd); +void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index 1cb2b8f30269c572bc76ddf9c04491d9b77bb3bc..fed0774346693fbd2d5ef725bfa620433664cee8 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -29,8 +29,8 @@ static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); -static void arbProcessBrokenLink(void *param); -static int32_t arbProcessPeerMsg(void *param, void *buffer); +static void arbProcessBrokenLink(int64_t rid); +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; static void * tsArbTcpPool; @@ -138,20 +138,20 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { sDebug("%s, arbitrator request is accepted", pNode->id); pNode->nodeFd = connFd; - pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, pNode, connFd); + pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd); return; } -static void arbProcessBrokenLink(void *param) { - SNodeConn *pNode = param; +static void arbProcessBrokenLink(int64_t rid) { + SNodeConn *pNode = (SNodeConn *)rid; sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno)); tfree(pNode); } -static int32_t arbProcessPeerMsg(void *param, void *buffer) { - SNodeConn *pNode = param; +static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { + SNodeConn *pNode = (SNodeConn *)rid; SSyncHead head; int32_t bytes = 0; char * cont = (char *)buffer; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index b43626ce43786a261a7caa94a1092a674a2df52f..436f4de0988aa87836172b8c0284ca767d27c5d8 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -35,19 +35,21 @@ char tsNodeFqdn[TSDB_FQDN_LEN] = {0}; static void * tsTcpPool = NULL; static void * tsSyncTmrCtrl = NULL; static void * tsVgIdHash = NULL; -static int32_t tsSyncRefId = -1; +static int32_t tsNodeRefId = -1; +static int32_t tsPeerRefId = -1; // local functions static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); static void syncRecoverFromMaster(SSyncPeer *pPeer); static void syncCheckPeerConnection(void *param, void *tmrId); -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); -static void syncProcessBrokenLink(void *param); -static int32_t syncProcessPeerMsg(void *param, void *buffer); +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); +static void syncProcessBrokenLink(int64_t rid); +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); +static void syncFreePeer(void *); static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); static void syncMonitorFwdInfos(void *param, void *tmrId); static void syncMonitorNodeRole(void *param, void *tmrId); @@ -55,7 +57,12 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t c static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); static void syncRestartPeer(SSyncPeer *pPeer); static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp); + static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); +static void syncStartCheckPeerConn(SSyncPeer *pPeer); +static void syncStopCheckPeerConn(SSyncPeer *pPeer); +static SSyncNode *syncAcquireNode(int64_t rid); +static void syncReleaseNode(SSyncNode *pNode); char* syncRole[] = { "offline", @@ -87,29 +94,34 @@ int32_t syncInit() { tsTcpPool = syncOpenTcpThreadPool(&info); if (tsTcpPool == NULL) { sError("failed to init tcpPool"); + syncCleanUp(); return -1; } tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); if (tsSyncTmrCtrl == NULL) { sError("failed to init tmrCtrl"); - syncCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; + syncCleanUp(); return -1; } tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVgIdHash == NULL) { sError("failed to init vgIdHash"); - taosTmrCleanUp(tsSyncTmrCtrl); - syncCloseTcpThreadPool(tsTcpPool); - tsTcpPool = NULL; - tsSyncTmrCtrl = NULL; + syncCleanUp(); return -1; } - tsSyncRefId = taosOpenRef(200, syncFreeNode); - if (tsSyncRefId < 0) { + tsNodeRefId = taosOpenRef(200, syncFreeNode); + if (tsNodeRefId < 0) { + sError("failed to init node ref"); + syncCleanUp(); + return -1; + } + + tsPeerRefId = taosOpenRef(1000, syncFreePeer); + if (tsPeerRefId < 0) { + sError("failed to init peer ref"); syncCleanUp(); return -1; } @@ -121,12 +133,12 @@ int32_t syncInit() { } void syncCleanUp() { - if (tsTcpPool) { + if (tsTcpPool != NULL) { syncCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; } - if (tsSyncTmrCtrl) { + if (tsSyncTmrCtrl != NULL) { taosTmrCleanUp(tsSyncTmrCtrl); tsSyncTmrCtrl = NULL; } @@ -136,8 +148,15 @@ void syncCleanUp() { tsVgIdHash = NULL; } - taosCloseRef(tsSyncRefId); - tsSyncRefId = -1; + if (tsNodeRefId != -1) { + taosCloseRef(tsNodeRefId); + tsNodeRefId = -1; + } + + if (tsPeerRefId != -1) { + taosCloseRef(tsPeerRefId); + tsPeerRefId = -1; + } sInfo("sync module is cleaned up"); } @@ -170,7 +189,8 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->quorum = pCfg->quorum; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - pNode->rid = taosAddRef(tsSyncRefId, pNode); + pNode->refCount = 1; + pNode->rid = taosAddRef(tsNodeRefId, pNode); if (pNode->rid < 0) { syncFreeNode(pNode); return -1; @@ -232,13 +252,18 @@ int64_t syncStart(const SSyncInfo *pInfo) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); + } + return pNode->rid; } void syncStop(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; sInfo("vgId:%d, cleanup sync", pNode->vgId); @@ -259,14 +284,14 @@ void syncStop(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); - taosRemoveRef(tsSyncRefId, rid); + syncReleaseNode(pNode); + taosRemoveRef(tsNodeRefId, rid); } int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { int32_t i, j; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, @@ -274,6 +299,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { pthread_mutex_lock(&pNode->mutex); + syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStopCheckPeerConn(pNode->peerInfo[index]); + } + for (i = 0; i < pNode->replica; ++i) { for (j = 0; j < pNewCfg->replica; ++j) { if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && @@ -330,28 +360,32 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } + syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb + for (int32_t index = 0; index < pNode->replica; ++index) { + syncStartCheckPeerConn(pNode->peerInfo[index]); + } + pthread_mutex_unlock(&pNode->mutex); sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); syncBroadcastStatus(pNode); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); return 0; } int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); - if (pNode == NULL) return 0; + SSyncNode *pNode = syncAcquireNode(rid); + if (pNode == NULL) return 0; int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); - taosReleaseRef(tsSyncRefId, rid); - + syncReleaseNode(pNode); return code; } void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncPeer *pPeer = pNode->pMaster; @@ -367,14 +401,14 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { } } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #if 0 void syncRecover(int64_t rid) { SSyncPeer *pPeer; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; // to do: add a few lines to check if recover is OK @@ -395,12 +429,12 @@ void syncRecover(int64_t rid) { pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } #endif int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return -1; pNodesRole->selfIndex = pNode->selfIndex; @@ -409,8 +443,7 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { pNodesRole->role[i] = pNode->peerInfo[i]->role; } - taosReleaseRef(tsSyncRefId, rid); - + syncReleaseNode(pNode); return 0; } @@ -445,6 +478,7 @@ static void syncAddArbitrator(SSyncNode *pNode) { static void syncFreeNode(void *param) { SSyncNode *pNode = param; + sDebug("vgId:%d, node is freed, refCount:%d", pNode->vgId, pNode->refCount); pthread_mutex_destroy(&pNode->mutex); tfree(pNode->pRecv); @@ -452,18 +486,50 @@ static void syncFreeNode(void *param) { tfree(pNode); } -void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); } +SSyncNode *syncAcquireNode(int64_t rid) { + SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid); + if (pNode == NULL) { + sDebug("failed to acquire node from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, acquire node refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount); + } + + return pNode; +} + +void syncReleaseNode(SSyncNode *pNode) { + int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1); + sTrace("vgId:%d, release node refId:%" PRId64 ", refCount:%d", pNode->vgId, pNode->rid, refCount); -int32_t syncDecPeerRef(SSyncPeer *pPeer) { - if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) { - taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); + taosReleaseRef(tsNodeRefId, pNode->rid); +} - sDebug("%s, resource is freed", pPeer->id); - tfree(pPeer); - return 0; +static void syncFreePeer(void *param) { + SSyncPeer *pPeer = param; + sDebug("%s, peer is freed, refCount:%d", pPeer->id, pPeer->refCount); + + syncReleaseNode(pPeer->pSyncNode); + tfree(pPeer); +} + +SSyncPeer *syncAcquirePeer(int64_t rid) { + SSyncPeer *pPeer = taosAcquireRef(tsPeerRefId, rid); + if (pPeer == NULL) { + sDebug("failed to acquire peer from refId:%" PRId64, rid); + } else { + int32_t refCount = atomic_add_fetch_32(&pPeer->refCount, 1); + sTrace("%s, acquire peer refId:%" PRId64 ", refCount:%d", pPeer->id, rid, refCount); } - return 1; + return pPeer; +} + +void syncReleasePeer(SSyncPeer *pPeer) { + int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1); + sTrace("%s, release peer refId:%" PRId64 ", refCount:%d", pPeer->id, pPeer->rid, refCount); + + taosReleaseRef(tsPeerRefId, pPeer->rid); } static void syncClosePeerConn(SSyncPeer *pPeer) { @@ -473,7 +539,8 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { taosClose(pPeer->syncFd); if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; - syncFreeTcpConn(pPeer->pConn); + void *pConn = pPeer->pConn; + if (pConn != NULL) syncFreeTcpConn(pPeer->pConn); } } @@ -482,11 +549,32 @@ static void syncRemovePeer(SSyncPeer *pPeer) { pPeer->ip = 0; syncClosePeerConn(pPeer); - syncDecPeerRef(pPeer); + //taosRemoveRef(tsPeerRefId, pPeer->rid); + syncReleasePeer(pPeer); +} + +static void syncStartCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; + + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); + if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { + int32_t checkMs = 100 + (pNode->vgId * 10) % 100; + if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; + sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); + taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); + } +} + +static void syncStopCheckPeerConn(SSyncPeer *pPeer) { + if (pPeer == NULL) return; + + taosTmrStopA(&pPeer->timer); + sDebug("%s, stop check peer connection", pPeer->id); } static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { - uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn); + uint32_t ip = taosGetIpv4FromFqdn(pInfo->nodeFqdn); if (ip == 0xFFFFFFFF) { sError("failed to add peer, can resolve fqdn:%s since %s", pInfo->nodeFqdn, strerror(errno)); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -508,17 +596,11 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->role = TAOS_SYNC_ROLE_OFFLINE; pPeer->pSyncNode = pNode; pPeer->refCount = 1; + pPeer->rid = taosAddRef(tsPeerRefId, pPeer); - sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port); - int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { - int32_t checkMs = 100 + (pNode->vgId * 10) % 100; - if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; - sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); - taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer); - } + sInfo("%s, %p it is configured, ep:%s:%u rid:%" PRId64, pPeer->id, pPeer, pPeer->fqdn, pPeer->port, pPeer->rid); - taosAcquireRef(tsSyncRefId, pNode->rid); + (void)syncAcquireNode(pNode->rid); return pPeer; } @@ -560,6 +642,9 @@ static void syncChooseMaster(SSyncNode *pNode) { index = i; } } + sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index); + } else { + sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica); } // add arbitrator connection @@ -580,6 +665,11 @@ static void syncChooseMaster(SSyncNode *pNode) { } } } + + if (index >= 0) { + sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId, + pNode->peerInfo[index]->id, index, onlineNum, replica); + } } if (index >= 0) { @@ -621,9 +711,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; + if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) { + sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } else { + nodeRole = TAOS_SYNC_ROLE_UNSYNCED; + sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); + } (*pNode->notifyRole)(pNode->vgId, nodeRole); - sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { for (int32_t index = 0; index < pNode->replica; ++index) { @@ -678,7 +772,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new if (pMaster) { // master is there pNode->pMaster = pMaster; - sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version); + sDebug("%s, it is the master, replica:%d sver:%" PRIu64, pMaster->id, pNode->replica, pMaster->version); if (syncValidateMaster(pPeer) < 0) return; @@ -711,10 +805,10 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } if (consistent) { - sDebug("vgId:%d, choose master", pNode->vgId); + sDebug("vgId:%d, choose master, replica:%d", pNode->vgId, pNode->replica); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inequality, replica:%d", pNode->vgId, pNode->replica); } } @@ -743,7 +837,7 @@ static void syncRestartPeer(SSyncPeer *pPeer) { int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { sDebug("%s, check peer connection in 1000 ms", pPeer->id); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } @@ -772,25 +866,30 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { } // start a new thread to retrieve the data - syncAddPeerRef(pPeer); + (void)syncAcquirePeer(pPeer->rid); + pthread_attr_t thattr; pthread_t thread; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); + int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); - if (ret != 0) { - sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno)); - syncDecPeerRef(pPeer); + if (ret < 0) { + sError("%s, failed to create sync retrieve thread since %s", pPeer->id, strerror(errno)); + syncReleasePeer(pPeer); } else { pPeer->sstatus = TAOS_SYNC_STATUS_START; - sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + sDebug("%s, sync retrieve thread:0x%08" PRIx64 " create successfully, rid:%" PRId64 ", set sstatus:%s", pPeer->id, + taosGetPthreadId(thread), pPeer->rid, syncStatus[pPeer->sstatus]); } } static void syncNotStarted(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -799,15 +898,22 @@ static void syncNotStarted(void *param, void *tmrId) { sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncTryRecoverFromMaster(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncRecoverFromMaster(SSyncPeer *pPeer) { @@ -823,7 +929,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { // Ensure the sync of mnode not interrupted if (pNode->vgId != 1 && tsSyncNum >= SYNC_MAX_NUM) { sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum); - taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -832,7 +938,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { SSyncMsg msg; syncBuildSyncReqMsg(&msg, pNode->vgId); - taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { sError("%s, failed to send sync-req to peer", pPeer->id); @@ -920,8 +1026,10 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead) { return 0; } -static int32_t syncProcessPeerMsg(void *param, void *buffer) { - SSyncPeer *pPeer = param; +static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return -1; + SSyncHead *pHead = buffer; SSyncNode *pNode = pPeer->pSyncNode; @@ -942,12 +1050,17 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) { } pthread_mutex_unlock(&pNode->mutex); + syncReleasePeer(pPeer); return code; } -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { - if (pPeer->peerFd < 0 || pPeer->ip == 0) return; +static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) { + if (pPeer->peerFd < 0 || pPeer->ip == 0) { + sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd); + syncRestartConnection(pPeer); + return -1; + } SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus msg; @@ -970,9 +1083,11 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pPeer->version, ack, tranId, statusType[type], pPeer->peerFd); + return 0; } else { sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); + return -1; } } @@ -989,7 +1104,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -1000,17 +1115,19 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, msg.tranId); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; - pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); taosClose(connFd); - taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } static void syncCheckPeerConnection(void *param, void *tmrId) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&pNode->mutex); @@ -1019,6 +1136,8 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { syncSetupPeerConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); + + syncReleasePeer(pPeer); } static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { @@ -1029,18 +1148,20 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - syncAddPeerRef(pPeer); - int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); + (void)syncAcquirePeer(pPeer->rid); + + int32_t ret = pthread_create(&thread, &thattr, (void *)syncRestoreData, (void *)pPeer->rid); pthread_attr_destroy(&thattr); if (ret < 0) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_INIT; - sError("%s, failed to create sync thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sError("%s, failed to create sync restore thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); + syncReleasePeer(pPeer); } else { - sInfo("%s, sync connection is up", pPeer->id); + sInfo("%s, sync restore thread:0x%08" PRIx64 " create successfully, rid:%" PRId64, pPeer->id, + taosGetPthreadId(thread), pPeer->rid); } } @@ -1073,7 +1194,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } - sDebug("vgId:%d, sync msg is received, tranId:%u", vgId, msg.tranId); + sDebug("vgId:%d, sync connection is incomming, tranId:%u", vgId, msg.tranId); SSyncNode *pNode = *ppNode; pthread_mutex_lock(&pNode->mutex); @@ -1101,8 +1222,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; - pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd); - syncAddPeerRef(pPeer); + pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd); sDebug("%s, ready to exchange data", pPeer->id); syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId()); } @@ -1111,23 +1231,21 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pthread_mutex_unlock(&pNode->mutex); } -static void syncProcessBrokenLink(void *param) { - if (param == NULL) return; // the connection for arbitrator - SSyncPeer *pPeer = param; +static void syncProcessBrokenLink(int64_t rid) { + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) return; + SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&pNode->mutex); sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); pPeer->peerFd = -1; - if (syncDecPeerRef(pPeer) != 0) { - syncRestartConnection(pPeer); - } - + syncRestartConnection(pPeer); pthread_mutex_unlock(&pNode->mutex); - taosReleaseRef(tsSyncRefId, pNode->rid); + + syncReleasePeer(pPeer); } static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { @@ -1198,7 +1316,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code static void syncMonitorNodeRole(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; for (int32_t index = 0; index < pNode->replica; index++) { @@ -1215,12 +1333,12 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { } pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static void syncMonitorFwdInfos(void *param, void *tmrId) { int64_t rid = (int64_t)param; - SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncFwds *pSyncFwds = pNode->pSyncFwds; @@ -1246,7 +1364,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); } - taosReleaseRef(tsSyncRefId, rid); + syncReleaseNode(pNode); } static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) { diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 8651879eb6cb06ff629c694f128b6107bf8f19a2..a5e268cdd262ee1cd4bae6433de9c7c764e6561a 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -90,15 +90,18 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { break; } + sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, + minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic); + // remove extra files on slave between the current and last index syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1); pindex = minfo.index; // check the file info sinfo = minfo; - sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size); - sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, - &sinfo.fversion); + sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); + sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, sinfo.name, + sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic); // if file not there or magic is not the same, file shall be synced memset(&fileAck, 0, sizeof(SFileAck)); @@ -116,6 +119,8 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { if (fileAck.sync == 0) { sDebug("%s, %s is the same", pPeer->id, minfo.name); continue; + } else { + sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } // if sync is required, open file, receive from master, and write to file @@ -155,7 +160,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { return code; } -static int32_t syncRestoreWal(SSyncPeer *pPeer) { +static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) { SSyncNode *pNode = pPeer->pSyncNode; int32_t ret, code = -1; uint64_t lastVer = 0; @@ -198,6 +203,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } free(pHead); + *wver = lastVer; return code; } @@ -321,12 +327,19 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeVersion = fversion; - sInfo("%s, start to restore wal", pPeer->id); - if (syncRestoreWal(pPeer) < 0) { - sError("%s, failed to restore wal", pPeer->id); + sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); + uint64_t wver = 0; + code = syncRestoreWal(pPeer, &wver); // lastwar + if (code < 0) { + sError("%s, failed to restore wal, code:%d", pPeer->id, code); return -1; } + if (wver != 0) { + nodeVersion = wver; + sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion); + } + nodeSStatus = TAOS_SYNC_STATUS_CACHE; sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { @@ -338,11 +351,18 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } void *syncRestoreData(void *param) { - SSyncPeer *pPeer = param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) { + sError("failed to restore data, invalid peer rid:%" PRId64, rid); + return NULL; + } + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); + sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); @@ -364,12 +384,15 @@ void *syncRestoreData(void *param) { (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; - sInfo("%s, sync over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); taosClose(pPeer->syncFd); syncCloseRecvBuffer(pNode); __sync_fetch_and_sub(&tsSyncNum, 1); - syncDecPeerRef(pPeer); + + // The ref is obtained in both the create thread and the current thread, so it is released twice + syncReleasePeer(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 02d990313e8c95518e22939c8ab0d41ad05604c2..d755ee9aa6f06d27eac144da67295ba02f98610a 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -104,7 +104,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); syncBuildFileInfo(&fileInfo, pNode->vgId); - sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); + sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, fileInfo.name, + fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic); // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo)); @@ -144,6 +145,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; sDebug("%s, %s is the same", pPeer->id, fileInfo.name); continue; + } else { + sDebug("%s, %s will be sent", pPeer->id, fileInfo.name); } // get the full path to file @@ -191,7 +194,7 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { } if (ret == 0) { - sTrace("sfd:%d, read to the end of file, ret:%d", sfd, ret); + sDebug("sfd:%d, read to the end of file, ret:%d", sfd, ret); return 0; } @@ -250,7 +253,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi break; } - sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); + sDebug("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); int32_t wsize = code; int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); @@ -461,9 +464,17 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { } void *syncRetrieveData(void *param) { - SSyncPeer *pPeer = (SSyncPeer *)param; + int64_t rid = (int64_t)param; + SSyncPeer *pPeer = syncAcquirePeer(rid); + if (pPeer == NULL) { + sError("failed to retrieve data, invalid peer rid:%" PRId64, rid); + return NULL; + } + SSyncNode *pNode = pPeer->pSyncNode; + taosBlockSIGPIPE(); + sInfo("%s, start to retrieve data, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); @@ -490,7 +501,12 @@ void *syncRetrieveData(void *param) { pPeer->fileChanged = 0; taosClose(pPeer->syncFd); - syncDecPeerRef(pPeer); + + // The ref is obtained in both the create thread and the current thread, so it is released twice + sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + + syncReleasePeer(pPeer); + syncReleasePeer(pPeer); return NULL; } diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 7bfdc4e440ee886065a4cd1b8f59d09e09556a47..474466673798ba027518a4bf5c3375492ce9fb60 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -42,7 +42,7 @@ typedef struct SPoolObj { typedef struct { SThreadObj *pThread; - void * ahandle; + int64_t handleId; int32_t fd; int32_t closedByApp; } SConnObj; @@ -112,7 +112,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -130,7 +130,7 @@ void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { pConn->fd = connFd; pConn->pThread = pThread; - pConn->ahandle = pPeer; + pConn->handleId = rid; pConn->closedByApp = 0; event.events = EPOLLIN | EPOLLRDHUP; @@ -164,7 +164,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { SPoolInfo * pInfo = &pPool->info; if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR); - (*pInfo->processBrokenLink)(pConn->ahandle); + (*pInfo->processBrokenLink)(pConn->handleId); pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); @@ -221,7 +221,7 @@ static void *syncProcessTcpData(void *param) { } if (pConn->closedByApp == 0) { - if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) { + if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) { syncFreeTcpConn(pConn); continue; } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5c978abd1da5cc5caa17450661d980ca648763c1..50679749036d7b38e124aad6980cdd44d40998a8 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -31,19 +31,19 @@ extern "C" { #endif -extern int tsdbDebugFlag; +extern int32_t tsdbDebugFlag; -#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} -#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} -#define tsdbWarn(...) { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} -#define tsdbInfo(...) { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} -#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} -#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) #define TSDB_MAX_TABLE_SCHEMAS 16 -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F -#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF #define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax))) @@ -66,7 +66,8 @@ typedef struct STable { SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void* eventHandler; // TODO void* streamHandler; // TODO - TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure + TSKEY lastKey; + SDataRow lastRow; char* sql; void* cqhandle; SRWLatch latch; // TODO: implementa latch functions @@ -360,8 +361,11 @@ typedef struct { #define TABLE_UID(t) (t)->tableId.uid #define TABLE_TID(t) (t)->tableId.tid #define TABLE_SUID(t) (t)->suid -#define TABLE_LASTKEY(t) (t)->lastKey #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) +#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch)) +#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch)) +#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch)) +#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch)) STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); void tsdbFreeMeta(STsdbMeta* pMeta); @@ -391,7 +395,7 @@ static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, STSchema* pSchema = NULL; STSchema* pTSchema = NULL; - if (lock) taosRLockLatch(&(pDTable->latch)); + if (lock) TSDB_RLOCK_TABLE(pDTable); if (version < 0) { // get the latest version of schema pTSchema = pDTable->schema[pDTable->numOfSchemas - 1]; } else { // get the schema with version @@ -413,7 +417,7 @@ static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, } _exit: - if (lock) taosRUnLockLatch(&(pDTable->latch)); + if (lock) TSDB_RUNLOCK_TABLE(pDTable); return pSchema; } @@ -433,6 +437,11 @@ static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { } } +static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { + ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); + return pTable->lastKey; +} + // ------------------ tsdbBuffer.c #define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 637b02cd32ae8ad8e4077684609dcac23922d8a0..696270d67086286dd3e9d86a69a5cc9cdc4b53eb 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -161,6 +161,11 @@ _err: static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); + SMemTable *pIMem = pRepo->imem; + tsdbLockRepo(pRepo); + pRepo->imem = NULL; + tsdbUnlockRepo(pRepo); + tsdbUnRefMemTable(pRepo, pIMem); sem_post(&(pRepo->readyToCommit)); } @@ -220,7 +225,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe SCommitIter *pIter = iters + tid; if (pIter->pTable == NULL) continue; - taosRLockLatch(&(pIter->pTable->latch)); + TSDB_RLOCK_TABLE(pIter->pTable); if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err; @@ -231,7 +236,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) { - taosRUnLockLatch(&(pIter->pTable->latch)); + TSDB_RUNLOCK_TABLE(pIter->pTable); tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable), tstrerror(terrno)); @@ -239,7 +244,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } } - taosRUnLockLatch(&(pIter->pTable->latch)); + TSDB_RUNLOCK_TABLE(pIter->pTable); // Move the last block to the new .l file if neccessary if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 9d65325001a33dedbf04cf7d065d1d803eeb4e1c..b34b2fa9e6d867527d98036da7972b50c5471c4b 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -77,9 +77,9 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { tsdbDebug( "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d daysPerFile %d keep " - "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d", + "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d update %d cacheLastRow %d", pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, - pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); + pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression, pCfg->update, pCfg->cacheLastRow); return 0; } @@ -281,6 +281,10 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { config.totalBlocks = pCfg->totalBlocks; configChanged = true; } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + config.cacheLastRow = pCfg->cacheLastRow; + configChanged = true; + } if (configChanged) { if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { @@ -475,6 +479,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { // update check if (pCfg->update != 0) pCfg->update = 1; + // update cacheLastRow + if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1; + return 0; _err: @@ -692,10 +699,12 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { } } -static int tsdbRestoreInfo(STsdbRepo *pRepo) { +static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pFGroup = NULL; + STsdbCfg * pCfg = &(pRepo->config); + SCompBlock *pBlock = NULL; SFileGroupIter iter; SRWHelper rhelper = {0}; @@ -713,7 +722,32 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err; SCompIdx *pIdx = &(rhelper.curCompIdx); - if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + if (pIdx->offset > 0 && lastKey < pIdx->maxKey) { + pTable->lastKey = pIdx->maxKey; + if (pCfg->cacheLastRow) { // load the block of data + if (tsdbLoadCompInfo(&rhelper, NULL) < 0) goto _err; + + pBlock = rhelper.pCompInfo->blocks + pIdx->numOfBlocks - 1; + if (tsdbLoadBlockData(&rhelper, pBlock, NULL) < 0) goto _err; + + // construct the data row + ASSERT(pTable->lastRow == NULL); + STSchema *pSchema = tsdbGetTableSchema(pTable); + pTable->lastRow = taosTMalloc(schemaTLen(pSchema)); + if (pTable->lastRow == NULL) { + goto _err; + } + + tdInitDataRow(pTable->lastRow, pSchema); + for (int icol = 0; icol < schemaNCols(pSchema); icol++) { + STColumn *pCol = schemaColAt(pSchema, icol); + SDataCol *pDataCol = rhelper.pDataCols[0]->cols + icol; + tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, + pCol->offset); + } + } + } } } @@ -800,6 +834,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->compression); tlen += taosEncodeFixedI8(buf, pCfg->update); + tlen += taosEncodeFixedI8(buf, pCfg->cacheLastRow); return tlen; } @@ -817,6 +852,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->compression)); buf = taosDecodeFixedI8(buf, &(pCfg->update)); + buf = taosDecodeFixedI8(buf, &(pCfg->cacheLastRow)); return buf; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 999e2deb413eb13cc7119ee9042a260366c5f9a3..07f001f68aecba589c68aa078fdb23dd071bbdd5 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -17,6 +17,7 @@ #include "tsdbMain.h" #define TSDB_DATA_SKIPLIST_LEVEL 5 +#define TSDB_MAX_INSERT_BATCH 512 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); static void tsdbFreeMemTable(SMemTable *pMemTable); @@ -35,6 +36,7 @@ static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPB static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter); static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter); +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row); static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, TSKEY now); @@ -205,10 +207,10 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int tsdbAsyncCommit(STsdbRepo *pRepo) { if (pRepo->mem == NULL) return 0; - SMemTable *pIMem = pRepo->imem; - sem_wait(&(pRepo->readyToCommit)); + ASSERT(pRepo->imem == NULL); + if (pRepo->code != TSDB_CODE_SUCCESS) { tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); } @@ -220,8 +222,6 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { tsdbScheduleCommit(pRepo); if (tsdbUnlockRepo(pRepo) < 0) return -1; - if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1; - return 0; } @@ -606,19 +606,13 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * STable * pTable = NULL; SSubmitBlkIter blkIter = {0}; SDataRow row = NULL; - void ** rows = NULL; + void * rows[TSDB_MAX_INSERT_BATCH] = {0}; int rowCounter = 0; ASSERT(pBlock->tid < pMeta->maxTables); pTable = pMeta->tables[pBlock->tid]; ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); - rows = (void **)calloc(pBlock->numOfRows, sizeof(void *)); - if (rows == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tsdbInitSubmitBlkIter(pBlock, &blkIter); while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) { @@ -632,9 +626,18 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * if (rows[rowCounter] != NULL) { rowCounter++; } + + if (rowCounter == TSDB_MAX_INSERT_BATCH) { + if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + goto _err; + } + + rowCounter = 0; + memset(rows, 0, sizeof(rows)); + } } - if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { + if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) { goto _err; } @@ -642,11 +645,9 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t * pRepo->stat.pointsWritten += points * schemaNCols(pSchema); pRepo->stat.totalStorage += points * schemaVLen(pSchema); - free(rows); return 0; _err: - free(rows); return -1; } @@ -663,9 +664,10 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void return -1; } - if (key > TABLE_LASTKEY(pTable)) { + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + if (key > lastKey) { tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, - REPO_ID(pRepo), key, TABLE_LASTKEY(pTable)); + REPO_ID(pRepo), key, lastKey); return 0; } } @@ -846,8 +848,10 @@ static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **ro if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]); pTableData->numOfRows += dsize; - // TODO: impl delete row thing - if (TABLE_LASTKEY(pTable) < dataRowKey(rows[rowCounter-1])) TABLE_LASTKEY(pTable) = dataRowKey(rows[rowCounter-1]); + // update table latest info + if (tsdbUpdateTableLatestInfo(pRepo, pTable, rows[rowCounter - 1]) < 0) { + return -1; + } return 0; } @@ -889,4 +893,38 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { } } } +} + +static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) { + STsdbCfg *pCfg = &pRepo->config; + + if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) { + if (pCfg->cacheLastRow || pTable->lastRow != NULL) { + SDataRow nrow = pTable->lastRow; + if (taosTSizeof(nrow) < dataRowLen(row)) { + SDataRow orow = nrow; + nrow = taosTMalloc(dataRowLen(row)); + if (nrow == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + dataRowCpy(nrow, row); + TSDB_WLOCK_TABLE(pTable); + pTable->lastKey = dataRowKey(row); + pTable->lastRow = nrow; + TSDB_WUNLOCK_TABLE(pTable); + taosTZfree(orow); + } else { + TSDB_WLOCK_TABLE(pTable); + pTable->lastKey = dataRowKey(row); + dataRowCpy(nrow, row); + TSDB_WUNLOCK_TABLE(pTable); + } + } else { + pTable->lastKey = dataRowKey(row); + } + } + + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 9dfa147c8f35c20cea9b85e38e38a3db60f3572b..7b08178f4949eff0c4e8802f1d04cf1c85554c5b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -377,11 +377,11 @@ int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { // Chage in memory if (pNewSchema != NULL) { // change super table tag schema - taosWLockLatch(&(pTable->pSuper->latch)); + TSDB_WLOCK_TABLE(pTable->pSuper); STSchema *pOldSchema = pTable->pSuper->tagSchema; pTable->pSuper->tagSchema = pNewSchema; tdFreeSchema(pOldSchema); - taosWUnLockLatch(&(pTable->pSuper->latch)); + TSDB_WUNLOCK_TABLE(pTable->pSuper); } bool isChangeIndexCol = (pMsg->colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))); @@ -392,9 +392,9 @@ int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { tsdbWLockRepoMeta(pRepo); tsdbRemoveTableFromIndex(pMeta, pTable); } - taosWLockLatch(&(pTable->latch)); + TSDB_WLOCK_TABLE(pTable); tdSetKVRowDataOfCol(&(pTable->tagVal), pMsg->colId, pMsg->type, POINTER_SHIFT(pMsg->data, pMsg->schemaLen)); - taosWUnLockLatch(&(pTable->latch)); + TSDB_WUNLOCK_TABLE(pTable); if (isChangeIndexCol) { tsdbAddTableIntoIndex(pMeta, pTable, false); tsdbUnlockRepoMeta(pRepo); @@ -587,7 +587,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, STable *pCTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; ASSERT(schemaVersion(pSchema) > schemaVersion(pCTable->schema[pCTable->numOfSchemas - 1])); - taosWLockLatch(&(pCTable->latch)); + TSDB_WLOCK_TABLE(pCTable); if (pCTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) { pCTable->schema[pCTable->numOfSchemas++] = pSchema; } else { @@ -599,7 +599,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); - taosWUnLockLatch(&(pCTable->latch)); + TSDB_WUNLOCK_TABLE(pCTable); if (insertAct) { int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pCTable); @@ -775,6 +775,7 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); + taosTZfree(pTable->lastRow); tfree(pTable->sql); free(pTable); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d5cc566b5541a41e863d4c136746019b4b172ce0..2444283435efeefdccdae2da5a047f23b7330ad5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -110,6 +110,7 @@ typedef struct STsdbQueryHandle { SArray* pTableCheckInfo; // SArray int32_t activeIndex; bool checkFiles; // check file stage + bool cachelastrow; // check if last row cached void* qinfo; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SFileGroup* pFileGroup; @@ -133,7 +134,9 @@ typedef struct STableGroupSupporter { STSchema* pTagSchema; } STableGroupSupporter; -static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); +static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); +static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); +static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); @@ -370,7 +373,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) { - pCond->twindow = changeTableGroupByLastrow(groupList); + pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table if (groupList->numOfTables == 0) { @@ -378,8 +381,14 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab } STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef); + int32_t code = checkForCachedLastRow(pQueryHandle, groupList); + if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 + terrno = code; + return NULL; + } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); + pQueryHandle->type = TSDB_QUERY_TYPE_LAST; return pQueryHandle; } @@ -956,9 +965,9 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } @@ -1704,7 +1713,32 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO return TSDB_CODE_SUCCESS; } -static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) { +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); + +static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + SQueryFilePos* cur = &pQueryHandle->cur; + + while(1) { + int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + + if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + // all data blocks in current file has been checked already, try next file if exists + return getFirstFileDataBlock(pQueryHandle, exists); + } else { // next block of the same file + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; + pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + } + } +} + +static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { pQueryHandle->numOfBlocks = 0; SQueryFilePos* cur = &pQueryHandle->cur; @@ -1789,7 +1823,23 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex cur->fid = pQueryHandle->pFileGroup->fileId; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo, exists); + return getDataBlockRv(pQueryHandle, pBlockInfo, exists); +} + +static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { + assert(cur != NULL && numOfBlocks > 0); + return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); +} + +static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + + SQueryFilePos* cur = &pQueryHandle->cur; + assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + + cur->slot += step; + cur->mixBlock = false; + cur->blockCompleted = false; } static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { @@ -1800,14 +1850,14 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists if (!pQueryHandle->locateStart) { pQueryHandle->locateStart = true; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); - return getDataBlocksInFilesImpl(pQueryHandle, exists); + return getFirstFileDataBlock(pQueryHandle, exists); } else { // check if current file block is all consumed STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; @@ -1815,27 +1865,26 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - // all data blocks in current file has been checked already, try next file if exists - return getDataBlocksInFilesImpl(pQueryHandle, exists); - } else { - // next block of the same file - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->slot += step; - - cur->mixBlock = false; - cur->blockCompleted = false; - - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); - } + // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, pQueryHandle->qinfo); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, + pQueryHandle->qinfo); int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = pQueryHandle->realNumOfRows > 0; + *exists = (pQueryHandle->realNumOfRows > 0); - return code; + if (code != TSDB_CODE_SUCCESS || *exists) { + return code; + } + } + + // current block is empty, try next block in file + // all data blocks in current file has been checked already, try next file if exists + if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { + return getFirstFileDataBlock(pQueryHandle, exists); + } else { + moveToNextDataBlockInCurrentFile(pQueryHandle); + STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pQueryHandle, pNext, exists); } } } @@ -2104,6 +2153,39 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { // restore the pMemRef pQueryHandle->pMemRef = pMemRef; return ret; + } else if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { + // the last row is cached in buffer, return it directly. + // here note that the pQueryHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + SQueryFilePos* cur = &pQueryHandle->cur; + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + + if (++pQueryHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); + if (ret != TSDB_CODE_SUCCESS) { + return false; + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj); + tfree(pRow); + + // update the last key value + pCheckInfo->lastKey = key + step; + + cur->rows = 1; // only one row + cur->lastKey = key + step; + cur->mixBlock = true; + cur->win.skey = key; + cur->win.ekey = key; + + return true; + } + + return false; } if (pQueryHandle->checkFiles) { @@ -2136,7 +2218,57 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return ret; } -STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { +/* + * 1. no data at all (pTable->lastKey = TSKEY_INITIAL_VAL), just return TSKEY_INITIAL_VAL + * 2. has data but not loaded, just return lastKey but not set pRes + * 3. has data and loaded, return lastKey and set pRes + */ +int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { + TSDB_RLOCK_TABLE(pTable); + *lastKey = pTable->lastKey; + + if ((*lastKey) != TSKEY_INITIAL_VAL && pTable->lastRow) { + *pRes = tdDataRowDup(pTable->lastRow); + if (*pRes == NULL) { + TSDB_RUNLOCK_TABLE(pTable); + return TSDB_CODE_TDB_OUT_OF_MEMORY; + } + } + + TSDB_RUNLOCK_TABLE(pTable); + return TSDB_CODE_SUCCESS; +} + +int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { + assert(pQueryHandle != NULL && groupList != NULL); + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + + SArray* group = taosArrayGetP(groupList->pGroupList, 0); + assert(group != NULL); + + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); + + int32_t code = tsdbGetCachedLastRow(pInfo->pTable, &pRow, &key); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->cachelastrow = false; + } else { + pQueryHandle->cachelastrow = (pRow != NULL); + } + + // update the tsdb query time range + if (pQueryHandle->cachelastrow) { + pQueryHandle->window = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = false; + pQueryHandle->activeIndex = -1; // start from -1 + } + + tfree(pRow); + return code; +} + +STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; int32_t totalNumOfTable = 0; @@ -2151,16 +2283,16 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { size_t numOfTables = taosArrayGetSize(pGroup); for(int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); // if the lastKey equals to INT64_MIN, there is no data in this table - TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; - keyInfo.pTable = pKeyInfo->pTable; + keyInfo.pTable = pInfo->pTable; keyInfo.lastKey = key; - pKeyInfo->lastKey = key; + pInfo->lastKey = key; if (key < window.skey) { window.skey = key; @@ -2174,11 +2306,11 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { // clear current group, unref unused table for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pKeyInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); // keyInfo.pTable may be NULL here. - if (pKeyInfo->pTable != keyInfo.pTable) { - tsdbUnRefTable(pKeyInfo->pTable); + if (pInfo->pTable != keyInfo.pTable) { + tsdbUnRefTable(pInfo->pTable); } } @@ -2513,13 +2645,12 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -static bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool tableFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = NULL; - + char* val = NULL; if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { val = (char*) TABLE_NAME(pTable); } else { @@ -2574,15 +2705,17 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) { return true; } +static void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param); + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the expression tree SExprTraverseSupp supp = { - .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp, + .nodeFilterFn = (__result_filter_fn_t) tableFilterFp, .setupInfoFn = filterPrepare, .pExtInfo = pSTable->tagSchema, }; - tExprTreeTraverse(pExpr, pSTable->pIndex, pRes, &supp); + getTableListfromSkipList(pExpr, pSTable->pIndex, pRes, &supp); tExprTreeDestroy(&pExpr, destroyHelper); return TSDB_CODE_SUCCESS; } @@ -2824,3 +2957,235 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); pGroupList->numOfTables = 0; } + +static void applyFilterToSkipListNode(SSkipList *pSkipList, tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + // Scan each node in the skiplist by using iterator + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + if (exprTreeApplayFilter(pExpr, pNode, param)) { + taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); + } + } + + tSkipListDestroyIter(iter); +} + +typedef struct { + char* v; + int32_t optr; +} SEndPoint; + +typedef struct { + SEndPoint* start; + SEndPoint* end; +} SQueryCond; + +// todo check for malloc failure +static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { + int32_t optr = queryColInfo->optr; + + if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || + optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { + pCond->start = calloc(1, sizeof(SEndPoint)); + pCond->start->optr = queryColInfo->optr; + pCond->start->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + pCond->end = calloc(1, sizeof(SEndPoint)); + pCond->end->optr = queryColInfo->optr; + pCond->end->v = queryColInfo->q; + } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) { + assert(0); + } + + return TSDB_CODE_SUCCESS; +} + +static void queryIndexedColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = NULL; + + SQueryCond cond = {0}; + if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) { + //todo handle error + } + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); + } else { + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); + } + + if (cond.start != NULL) { + int32_t optr = cond.start->optr; + + if (optr == TSDB_RELATION_EQUAL) { // equals + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + int32_t ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + if (ret != 0) { + break; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal + bool comp = true; + int32_t ret = 0; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v); + assert(ret >= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_GREATER) { + continue; + } else { + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; + } + } + } else if (optr == TSDB_RELATION_NOT_EQUAL) { // not equal + bool comp = true; + + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + tSkipListDestroyIter(iter); + + comp = true; + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); + while(tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); + if (comp) { + continue; + } + + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + + } else { + assert(0); + } + } else { + int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID; + if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { + bool comp = true; + int32_t ret = 0; + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + if (comp) { + ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v); + assert(ret <= 0); + } + + if (ret == 0 && optr == TSDB_RELATION_LESS) { + continue; + } else { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + comp = false; // no need to compare anymore + } + } + } else { + assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL); + + while (tSkipListIterNext(iter)) { + SSkipListNode *pNode = tSkipListIterGet(iter); + + bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); + if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || + (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); + } + } + } + } + + free(cond.start); + free(cond.end); + tSkipListDestroyIter(iter); +} + +static void queryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + + char *pData = SL_GET_NODE_DATA(pNode); + tstr *name = (tstr*) tsdbGetTableName((void*) pData); + + // todo speed up by using hash + if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(name, pQueryInfo->q); + } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = !pQueryInfo->compare(name, pQueryInfo->q); + } + } else { + addToResult = filterFp(pNode, pQueryInfo); + } + + if (addToResult) { + STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(res, &info); + } + } + + tSkipListDestroyIter(iter); +} + +// Apply the filter expression to each node in the skiplist to acquire the qualified nodes in skip list +void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param) { + if (pExpr == NULL) { + return; + } + + tExprNode *pLeft = pExpr->_node.pLeft; + tExprNode *pRight = pExpr->_node.pRight; + + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { + assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY)); + + param->setupInfoFn(pExpr, param->pExtInfo); + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { + queryIndexedColumn(pSkipList, pQueryInfo, result); + } else { + queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); + } + + return; + } + + // The value of hasPK is always 0. + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + assert(weight == 0 && pSkipList != NULL && taosArrayGetSize(result) == 0); + + //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes + applyFilterToSkipListNode(pSkipList, pExpr, result, param); +} diff --git a/src/util/inc/buildInfo.h b/src/util/inc/buildInfo.h deleted file mode 100644 index 8d169d618d51a1e0a4191c8c8fbe370536970478..0000000000000000000000000000000000000000 --- a/src/util/inc/buildInfo.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef _TS_BUILD_H_ -#define _TS_BUILD_H_ - -extern const char tsVersion[]; -extern const char tsBuildInfo[]; - -#endif diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 5bada93d1c77462fea0d16708fef2546976a8d99..b6b49693f60b15a51eaa35c95555d0f35d36f017 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -32,17 +32,18 @@ typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t keyLen; // length of the key - size_t dataLen; // length of data - int8_t count; // reference count - int8_t removed; // flag to indicate removed + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + int8_t removed; // flag to indicate removed + int8_t count; // reference count char data[]; } SHashNode; #define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) #define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) #define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode)); + typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -115,7 +116,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param dsize * @return */ -void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize); +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize); /** * remove item with the specified key @@ -129,16 +130,14 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); +void taosHashEmpty(SHashObj *pHashObj); + /** * clean up hash table * @param handle */ void taosHashCleanup(SHashObj *pHashObj); -/* -void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *); -*/ - /** * * @param pHashObj diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 665528f14058cdf4edb3c262a285dbbd3482dc26..bc1da9858a3ea66610c6d03364d753b6b2f06313 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -41,6 +41,7 @@ enum { }; enum { + TAOS_CFG_VTYPE_INT8, TAOS_CFG_VTYPE_INT16, TAOS_CFG_VTYPE_INT32, TAOS_CFG_VTYPE_FLOAT, diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index cd5092f30a290c51de49e38b0226bbed637dc0e6..085c10c55198fc92bfacb0628c38f05da508bfb9 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -52,6 +52,8 @@ void *taosIterateRef(int rsetId, int64_t rid); // return the number of references in system int taosListRef(); +#define RID_VALID(x) ((x) > 0) + /* sample code to iterate the refs void demoIterateRefs(int rsetId) { diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index 391cc44accadd785fb46e02aa79cde7617834c70..a339955cc02f31607c63b5c473c57c4d0e6f6f1c 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -33,7 +33,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(SOCKET sockFd); int32_t taosGetFqdn(char *); -uint32_t taosGetIpFromFqdn(const char *); +uint32_t taosGetIpv4FromFqdn(const char *); void tinet_ntoa(char *ipstr, uint32_t ip); uint32_t ip2uint(const char *const ip_addr); diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 7a835e87e753d98ec19809e4f3eec32bbe37e55a..1a9c6d314b3fd8a6b3ad6928b6faa4f7fad32757 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -271,10 +271,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashGetCB(pHashObj, key, keyLen, NULL, NULL, 0); + return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL, 0); } -void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { return NULL; } @@ -313,10 +313,10 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f } if (d != NULL) { - memcpy(d, GET_HASH_NODE_DATA(pNode), dsize); - } else { - data = GET_HASH_NODE_DATA(pNode); + memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); } + + data = GET_HASH_NODE_DATA(pNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -472,38 +472,49 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi return 0; } -void taosHashCleanup(SHashObj *pHashObj) { +void taosHashEmpty(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } + uDebug("hash:%p cleanup hash table", pHashObj); + SHashNode *pNode, *pNext; __wr_lock(&pHashObj->lock, pHashObj->type); - if (pHashObj->hashList) { - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num == 0) { - assert(pEntry->next == 0); - continue; - } + for (int32_t i = 0; i < pHashObj->capacity; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + assert(pEntry->next == 0); + continue; + } - pNode = pEntry->next; - assert(pNode != NULL); + pNode = pEntry->next; + assert(pNode != NULL); - while (pNode) { - pNext = pNode->next; - FREE_HASH_NODE(pHashObj, pNode); + while (pNode) { + pNext = pNode->next; + FREE_HASH_NODE(pHashObj, pNode); - pNode = pNext; - } + pNode = pNext; } - free(pHashObj->hashList); + pEntry->num = 0; + pEntry->next = NULL; } + pHashObj->size = 0; __wr_unlock(&pHashObj->lock, pHashObj->type); +} + +void taosHashCleanup(SHashObj *pHashObj) { + if (pHashObj == NULL) { + return; + } + + taosHashEmpty(pHashObj); + tfree(pHashObj->hashList); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); @@ -654,7 +665,7 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; - pNewNode->dataLen = dsize; + pNewNode->dataLen = (uint32_t) dsize; pNewNode->count = 1; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2571f11ba41b653e1fcc639cf1c22ef1f50b3448..3afdf41d05cf131922a24c6641d02b95acfe2a93 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -278,7 +278,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen } SCacheDataNode* ptNode = NULL; - taosHashGetCB(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode, sizeof(void*)); + taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode, sizeof(void*)); void* pData = (ptNode != NULL)? ptNode->data:NULL; @@ -547,13 +547,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { return; } + __cache_wr_lock(pCacheObj); STrashElem *pElem = calloc(1, sizeof(STrashElem)); pElem->pData = pNode; - pElem->prev = NULL; + pElem->prev = NULL; + pElem->next = NULL; pNode->inTrashcan = true; pNode->pTNodeHeader = pElem; - __cache_wr_lock(pCacheObj); pElem->next = pCacheObj->pTrash; if (pCacheObj->pTrash) { pCacheObj->pTrash->prev = pElem; @@ -563,8 +564,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pCacheObj->numOfElemsInTrash++; __cache_unlock(pCacheObj); - uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, - pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash); + uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, + pNode->data, pElem, pCacheObj->numOfElemsInTrash); } void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 1d2bd6252f528c938d27aacb41b76b52bf2b334b..0a9f5a98c06c80efc123c6a05f4811b6c66c2615 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -95,6 +95,23 @@ static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { } } +static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { + int32_t value = atoi(input_value); + int8_t *option = (int8_t *)cfg->ptr; + if (value < cfg->minValue || value > cfg->maxValue) { + uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", + cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); + } else { + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { + *option = (int8_t)value; + cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; + } else { + uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, + tsCfgStatusStr[cfg->cfgStatus], *option); + } + } +} + static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { int length = (int)strlen(input_value); char *option = (char *)cfg->ptr; @@ -204,6 +221,9 @@ static void taosReadConfigOption(const char *option, char *value) { if (strcasecmp(cfg->option, option) != 0) continue; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + taosReadInt8Config(cfg, value); + break; case TAOS_CFG_VTYPE_INT16: taosReadInt16Config(cfg, value); break; @@ -376,6 +396,9 @@ void taosPrintGlobalCfg() { blank[blankLen] = 0; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_INT16: uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; @@ -408,6 +431,9 @@ static void taosDumpCfg(SGlobalCfg *cfg) { blank[blankLen] = 0; switch (cfg->valType) { + case TAOS_CFG_VTYPE_INT8: + printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_INT16: printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index c0f89e8465c35611cdf6437d8f8a64572ddc2d6b..e0fe51e22a6405273cbb60fbfa04d3a5956105b0 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -64,10 +64,10 @@ typedef struct { } SLogObj; int32_t tsLogKeepDays = 0; -int32_t tsAsyncLog = 1; +int8_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 0.1f; +float tsMinimalLogDirGB = 1.0f; #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; #else @@ -364,7 +364,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { ptm = localtime_r(&curTime, &Tm); len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); @@ -450,7 +450,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . ptm = localtime_r(&curTime, &Tm); len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index 89601147a5923ac413800bdb6d134b266f0663b9..e1b834b9499311d6932e5778a0e8116756a11961 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -449,7 +449,7 @@ static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { int32_t endPort = startPort + 11; uInfo("work as client, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); - uint32_t serverIp = taosGetIpFromFqdn(host); + uint32_t serverIp = taosGetIpv4FromFqdn(host); if (serverIp == 0xFFFFFFFF) { uError("failed to resolve fqdn:%s", host); exit(-1); diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index f2db0b331619cac89aae14fe38d9cb0e65f9c034..56f2322a7bd1e0778e2d52667663b3baf5c08c1f 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -249,7 +249,7 @@ void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetPthreadId()); + ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); va_start(argpointer, format); len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); va_end(argpointer); diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 1f83abcb847147da1d9ad0d17d26081176df0c93..3ef45e9b191fddc45220c7b07551b098ae0891ba 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -447,7 +447,7 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { } static void taosLockList(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { @@ -457,7 +457,7 @@ static void taosLockList(int64_t *lockedBy) { } static void taosUnlockList(int64_t *lockedBy) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(lockedBy, tid, 0) != tid) { assert(false); } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 7c950b7999b627d9334eae37eab35e1796c804dc..e3798162e8e949d65c6c6382574a43d94a474ee6 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -280,10 +280,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) { tSkipListRLock(pSkipList); if (iter->order == TSDB_ORDER_ASC) { - if (iter->cur == pSkipList->pTail) { + // no data in the skip list + if (iter->cur == pSkipList->pTail || iter->next == NULL) { + iter->cur = pSkipList->pTail; tSkipListUnlock(pSkipList); return false; } + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it @@ -295,9 +298,11 @@ bool tSkipListIterNext(SSkipListIterator *iter) { iter->step++; } else { if (iter->cur == pSkipList->pHead) { + iter->cur = pSkipList->pHead; tSkipListUnlock(pSkipList); return false; } + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 2543c8170896825ae2a94349c0218e7d99ce8592..1a5c3bd64da3274773196da4a03c8c0613810f89 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -18,6 +18,10 @@ #include "tsocket.h" #include "taoserror.h" +#ifndef SIGPIPE + #define SIGPIPE EPIPE +#endif + int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; @@ -40,9 +44,9 @@ int32_t taosGetFqdn(char *fqdn) { return 0; } -uint32_t taosGetIpFromFqdn(const char *fqdn) { +uint32_t taosGetIpv4FromFqdn(const char *fqdn) { struct addrinfo hints = {0}; - hints.ai_family = AF_UNSPEC; + hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; struct addrinfo *result = NULL; @@ -115,6 +119,10 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nwritten; ptr += nwritten; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); @@ -142,6 +150,10 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { nleft -= nread; ptr += nread; } + + if (errno == SIGPIPE || errno == EPIPE) { + return -1; + } } return (nbytes - nleft); diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 114f0764e8f50dcada076f657be5db126a4eb1f0..015c687b3d941db3f6414d0d3d92564dd14e0fdb 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -19,7 +19,7 @@ #include "ttimer.h" #include "tutil.h" -extern uint32_t tscEmbedded; +extern int8_t tscEmbedded; #define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} #define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tscEmbedded ? 255 : tmrDebugFlag, __VA_ARGS__); }} @@ -119,7 +119,7 @@ static void timerDecRef(tmr_obj_t* timer) { } static void lockTimerList(timer_list_t* list) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); int i = 0; while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { @@ -129,7 +129,7 @@ static void lockTimerList(timer_list_t* list) { } static void unlockTimerList(timer_list_t* list) { - int64_t tid = taosGetPthreadId(); + int64_t tid = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&(list->lockedBy), tid, 0) != tid) { assert(false); tmrError("%" PRId64 " trying to unlock a timer list not locked by current thread.", tid); @@ -257,7 +257,7 @@ static bool removeFromWheel(tmr_obj_t* timer) { static void processExpiredTimer(void* handle, void* arg) { tmr_obj_t* timer = (tmr_obj_t*)handle; - timer->executedBy = taosGetPthreadId(); + timer->executedBy = taosGetSelfPthreadId(); uint8_t state = atomic_val_compare_exchange_8(&timer->state, TIMER_STATE_WAITING, TIMER_STATE_EXPIRED); if (state == TIMER_STATE_WAITING) { const char* fmt = "%s timer[id=%" PRIuPTR ", fp=%p, param=%p] execution start."; @@ -406,7 +406,7 @@ static bool doStopTimer(tmr_obj_t* timer, uint8_t state) { return false; } - if (timer->executedBy == taosGetPthreadId()) { + if (timer->executedBy == taosGetSelfPthreadId()) { // taosTmrReset is called in the timer callback, should do nothing in this // case to avoid dead lock. note taosTmrReset must be the last statement // of the callback funtion, will be a bug otherwise. diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index fb1868ab13cd677542d0a5a70d48e8b8ee2df829..7adeda590a509df9554c5dc83fa66758ed8ed9d2 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -45,6 +45,9 @@ typedef struct { int8_t accessState; int8_t isFull; int8_t isCommiting; + int8_t dbReplica; + int8_t dropped; + int8_t reserved; uint64_t version; // current version uint64_t cversion; // version while commit start uint64_t fversion; // version on saved data file @@ -64,7 +67,6 @@ typedef struct { void * qMgmt; char * rootDir; tsem_t sem; - int8_t dropped; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; pthread_mutex_t statusMutex; } SVnodeObj; diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index aff7315d285e0283f7c427ec3677764ffa5ebcf8..0b32f97939a04b695c641a076e863fe37d85a437 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -34,11 +34,13 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow; pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP; - pVnode->syncCfg.replica = vnodeMsg->cfg.replications; + pVnode->syncCfg.replica = vnodeMsg->cfg.vgReplica; pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum; + pVnode->dbReplica = vnodeMsg->cfg.dbReplica; for (int i = 0; i < pVnode->syncCfg.replica; ++i) { SVnodeDesc *node = &vnodeMsg->nodes[i]; @@ -106,9 +108,10 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion"); if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file); - goto PARSE_VCFG_ERROR; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; } - vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) { @@ -201,12 +204,21 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.wals = (int8_t)wals->valueint; - cJSON *replica = cJSON_GetObjectItem(root, "replica"); - if (!replica || replica->type != cJSON_Number) { + cJSON *vgReplica = cJSON_GetObjectItem(root, "replica"); + if (!vgReplica || vgReplica->type != cJSON_Number) { vError("vgId:%d, failed to read %s, replica not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.replications = (int8_t)replica->valueint; + vnodeMsg.cfg.vgReplica = (int8_t)vgReplica->valueint; + + cJSON *dbReplica = cJSON_GetObjectItem(root, "dbReplica"); + if (!dbReplica || dbReplica->type != cJSON_Number) { + vError("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file); + vnodeMsg.cfg.dbReplica = vnodeMsg.cfg.vgReplica; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.dbReplica = (int8_t)dbReplica->valueint; + } cJSON *quorum = cJSON_GetObjectItem(root, "quorum"); if (!quorum || quorum->type != cJSON_Number) { @@ -215,6 +227,15 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow"); + if (!cacheLastRow || cacheLastRow->type != cJSON_Number) { + vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file); + vnodeMsg.cfg.cacheLastRow = 0; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.cacheLastRow = (int8_t)cacheLastRow->valueint; + } + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file); @@ -222,7 +243,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } int size = cJSON_GetArraySize(nodeInfos); - if (size != vnodeMsg.cfg.replications) { + if (size != vnodeMsg.cfg.vgReplica) { vError("vgId:%d, failed to read %s, nodeInfos size not matched", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } @@ -300,16 +321,18 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pMsg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pMsg->cfg.walLevel); len += snprintf(content + len, maxLen - len, " \"fsync\": %d,\n", pMsg->cfg.fsyncPeriod); - len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.replications); + len += snprintf(content + len, maxLen - len, " \"replica\": %d,\n", pMsg->cfg.vgReplica); + len += snprintf(content + len, maxLen - len, " \"dbReplica\": %d,\n", pMsg->cfg.dbReplica); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); - for (int32_t i = 0; i < pMsg->cfg.replications; i++) { + for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) { SVnodeDesc *node = &pMsg->nodes[i]; dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL); len += snprintf(content + len, maxLen - len, " \"nodeId\": %d,\n", node->nodeId); len += snprintf(content + len, maxLen - len, " \"nodeEp\": \"%s\"\n", node->nodeEp); - if (i < pMsg->cfg.replications - 1) { + if (i < pMsg->cfg.vgReplica - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); } else { len += snprintf(content + len, maxLen - len, " }]\n"); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index b1815e959d37a3a0d0e3484ff6a485fddc468a8d..8f44a248ced6157195955fdf79ec713ea6f25072 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -86,6 +86,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.compression = pVnodeCfg->cfg.compression; tsdbCfg.update = pVnodeCfg->cfg.update; + tsdbCfg.cacheLastRow = pVnodeCfg->cfg.cacheLastRow; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index ee5aa5ca90c4c0612c98468cfae67ac3e1d17985..8469ab12c1ef5833e73058c167afb2777ac76ba9 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -89,7 +89,10 @@ static void vnodeIncRef(void *ptNode) { } void *vnodeAcquire(int32_t vgId) { - SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + SVnodeObj **ppVnode = NULL; + if (tsVnodesHash != NULL) { + ppVnode = taosHashGetClone(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + } if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; @@ -139,6 +142,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { pLoad->totalStorage = htobe64(totalStorage); pLoad->compStorage = htobe64(compStorage); pLoad->pointsWritten = htobe64(pointsWritten); + pLoad->vnodeVersion = htobe64(pVnode->version); pLoad->status = pVnode->status; pLoad->role = pVnode->role; pLoad->replica = pVnode->syncCfg.replica; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 637d470f8a775706b1d04799af83e741a81b0ee4..a972ffec1cb5279b191dc1256b85c28e0e5cd9eb 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -65,13 +65,17 @@ static int32_t vnodeCheckRead(SVnodeObj *pVnode) { return TSDB_CODE_APP_NOT_READY; } - if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica, - syncRole[pVnode->role], pVnode->refCount, pVnode); - return TSDB_CODE_APP_NOT_READY; + if (pVnode->role == TAOS_SYNC_ROLE_MASTER) { + return TSDB_CODE_SUCCESS; + } + + if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) { + return TSDB_CODE_SUCCESS; } - return TSDB_CODE_SUCCESS; + vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica, + syncRole[pVnode->role], pVnode->refCount, pVnode); + return TSDB_CODE_APP_NOT_READY; } void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) { @@ -303,8 +307,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; } else { - void *h1 = qGetResultRetrieveMsg(*qhandle); - assert(h1 == NULL); + //void *h1 = qGetResultRetrieveMsg(*qhandle); + + /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */ + //assert(h1 == NULL); + freehandle = qQueryCompleted(*qhandle); } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 80e2dc422e07395f19fa2cad79b37037e3d06f86..571c7d667adfd285387ab0c11a308dc7472bb0af 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -101,13 +101,19 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara return syncCode; } -static int32_t vnodeCheckWrite(void *vparam) { - SVnodeObj *pVnode = vparam; +static int32_t vnodeCheckWrite(SVnodeObj *pVnode) { if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { vDebug("vgId:%d, no write auth, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); return TSDB_CODE_VND_NO_WRITE_AUTH; } + if (pVnode->dbReplica != pVnode->syncCfg.replica && + pVnode->syncCfg.nodeInfo[pVnode->syncCfg.replica - 1].nodeId == dnodeGetDnodeId()) { + vDebug("vgId:%d, vnode is balancing and will be dropped, dbReplica:%d vgReplica:%d, refCount:%d pVnode:%p", + pVnode->vgId, pVnode->dbReplica, pVnode->syncCfg.replica, pVnode->refCount, pVnode); + return TSDB_CODE_VND_IS_BALANCING; + } + // tsdb may be in reset state if (pVnode->tsdb == NULL) { vDebug("vgId:%d, tsdb is null, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); @@ -209,29 +215,21 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR return TSDB_CODE_SUCCESS; } -int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { - SVnodeObj *pVnode = vparam; - SWalHead * pHead = wparam; - int32_t code = 0; - - if (qtype == TAOS_QTYPE_RPC) { - code = vnodeCheckWrite(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - } - +static SVWriteMsg *vnodeBuildVWriteMsg(SVnodeObj *pVnode, SWalHead *pHead, int32_t qtype, SRpcMsg *pRpcMsg) { if (pHead->len > TSDB_MAX_WAL_SIZE) { vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version); - return TSDB_CODE_WAL_SIZE_LIMIT; + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + return NULL; } int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len; SVWriteMsg *pWrite = taosAllocateQitem(size); if (pWrite == NULL) { - return TSDB_CODE_VND_OUT_OF_MEMORY; + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return NULL; } - if (rparam != NULL) { - SRpcMsg *pRpcMsg = rparam; + if (pRpcMsg != NULL) { pWrite->rpcMsg = *pRpcMsg; } @@ -241,6 +239,21 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar atomic_add_fetch_32(&pVnode->refCount, 1); + return pWrite; +} + +static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) { + SVnodeObj *pVnode = pWrite->pVnode; + + if (pWrite->qtype == TAOS_QTYPE_RPC) { + int32_t code = vnodeCheckWrite(pVnode); + if (code != TSDB_CODE_SUCCESS) { + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + return code; + } + } + int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); if (queued > MAX_QUEUED_MSG_NUM) { int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3; @@ -249,15 +262,25 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar taosMsleep(ms); } - code = vnodePerformFlowCtrl(pWrite); - if (code != 0) return 0; - vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); - taosWriteQitem(pVnode->wqueue, qtype, pWrite); + taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); return TSDB_CODE_SUCCESS; } +int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { + SVWriteMsg *pWrite = vnodeBuildVWriteMsg(vparam, wparam, qtype, rparam); + if (pWrite == NULL) { + assert(terrno != 0); + return terrno; + } + + int32_t code = vnodePerformFlowCtrl(pWrite); + if (code != 0) return 0; + + return vnodeWriteToWQueueImp(pWrite); +} + void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { SVnodeObj *pVnode = vparam; @@ -271,7 +294,7 @@ void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { SVWriteMsg *pWrite = param; SVnodeObj * pVnode = pWrite->pVnode; - int32_t code = TSDB_CODE_VND_SYNCING; + int32_t code = TSDB_CODE_VND_IS_SYNCING; if (pVnode->flowctrlLevel <= 0) code = TSDB_CODE_VND_IS_FLOWCTRL; @@ -287,7 +310,10 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->processedCount); pWrite->processedCount = 0; - taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); + code = vnodeWriteToWQueueImp(pWrite); + if (code != 0) { + dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } } } } @@ -297,7 +323,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; - if (tsFlowCtrl == 0) { + if (tsEnableFlowCtrl == 0) { int32_t ms = pow(2, pVnode->flowctrlLevel + 2); if (ms > 100) ms = 100; vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); @@ -311,4 +337,4 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { pWrite->processedCount); return TSDB_CODE_VND_ACTION_IN_PROGRESS; } -} +} \ No newline at end of file diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index aae81c19f41a94874b1044db2fdd5e2950cb21e4..09547710c6891c4f08d594b50298b3c52e89a45b 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -42,6 +42,7 @@ pipeline { pre_test() sh ''' cd ${WKC}/tests + find pytest -name '*'sql|xargs rm -rf ./test-all.sh pytest date''' } @@ -50,12 +51,7 @@ pipeline { agent{label 'master'} steps { pre_test() - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/tests/pytest - python3 concurrent_inquiry.py -c 1 - ''' - } + sh ''' cd ${WKC}/tests ./test-all.sh b1 @@ -82,53 +78,26 @@ pipeline { ./handle_crash_gen_val_log.sh ''' } - sh ''' - cd ${WKC}/tests - ./test-all.sh b2 - date - ''' - } - } - - stage('test_valgrind') { - agent{label "186"} - - steps { - pre_test() - sh ''' - cd ${WKC}/tests/pytest - ./valgrind-test.sh 2>&1 > mem-error-out.log - ./handle_val_log.sh - - date - cd ${WKC}/tests - ./test-all.sh b3 - date''' - } - } - stage('connector'){ - agent{label "release"} - steps{ sh''' - cd ${WORKSPACE} - git checkout develop + systemctl start taosd + sleep 10 ''' catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' - cd ${WORKSPACE}/tests/gotest + cd ${WKC}/tests/gotest bash batchtest.sh ''' } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' - cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker + cd ${WKC}/tests/examples/python/PYTHONConnectorChecker python3 PythonChecker.py ''' } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' - cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ - mvn clean package assembly:single >/dev/null + cd ${WKC}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single -DskipTests >/dev/null java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 ''' } @@ -138,9 +107,41 @@ pipeline { dotnet run ''' } + sh ''' + systemctl stop taosd + cd ${WKC}/tests + ./test-all.sh b2 + date + ''' + } + } + + stage('test_valgrind') { + agent{label "186"} + + steps { + pre_test() + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + nohup taosd >/dev/null & + sleep 10 + python3 concurrent_inquiry.py -c 1 + + ''' + } + sh ''' + cd ${WKC}/tests/pytest + ./valgrind-test.sh 2>&1 > mem-error-out.log + ./handle_val_log.sh - } - } + date + cd ${WKC}/tests + ./test-all.sh b3 + date''' + } + } + stage('arm64_build'){ agent{label 'arm64'} steps{ diff --git a/tests/examples/C#/taosdemo/README.md b/tests/examples/C#/taosdemo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..09e1d659b74f22670c6bfe1e6fa85c7a457d6b13 --- /dev/null +++ b/tests/examples/C#/taosdemo/README.md @@ -0,0 +1,37 @@ +install build environment +=== +yum/apt install mono-complete + +build C# version taosdemo +=== +mcs -out:taosdemo *.cs + +run C# version taosdemo +=== +Usage: mono taosdemo.exe [OPTION...] + + -h host, The host to connect to TDengine. Default is localhost. + -p port, The TCP/IP port number to use for the connection. Default is 0. + -u user, The user name to use when connecting to the server. Default is 'root'. + -P password, The password to use when connecting to the server. Default is 'taosdata'. + -d database, Destination database. Default is 'test'. + -a replica, Set the replica parameters of the database, Default 1, min: 1, max: 3. + -m table_prefix, Table prefix name. Default is 't'. + -s sql file, The select sql file. + -M stable, Use super table. + -o outputfile, Direct output to the named file. Default is './output.txt'. + -q query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC. + -b type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'. + -w length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8 + -l num_of_cols_per_record, The number of columns per record. Default is 3. + -T num_of_threads, The number of threads. Default is 10. + -r num_of_records_per_req, The number of records per request. Default is 1000. + -t num_of_tables, The number of tables. Default is 10000. + -n num_of_records_per_table, The number of records per table. Default is 10000. + -c config_directory, Configuration directory. Default is '/etc/taos/'. + -x flag, Insert only flag. + -O order, Insert mode--0: In order, 1: Out of order. Default is in order. + -R rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50. + -D Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database. + -v Print verbose output + -y Skip read key for continous test, default is not skip diff --git a/tests/examples/C#/taosdemo/TDengineDriver.cs b/tests/examples/C#/taosdemo/TDengineDriver.cs new file mode 120000 index 0000000000000000000000000000000000000000..9bee9fb271a995d8ecec6a07fac2861b2a6f200f --- /dev/null +++ b/tests/examples/C#/taosdemo/TDengineDriver.cs @@ -0,0 +1 @@ +../../../../src/connector/C#/TDengineDriver.cs \ No newline at end of file diff --git a/tests/examples/C#/taosdemo/taosdemo.cs b/tests/examples/C#/taosdemo/taosdemo.cs new file mode 100644 index 0000000000000000000000000000000000000000..8e48fa2c8f2d238b7aebf8cba6bf455cb6b890be --- /dev/null +++ b/tests/examples/C#/taosdemo/taosdemo.cs @@ -0,0 +1,750 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +using System; +using System.Text; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Collections; +using System.Threading; +using System.Diagnostics; + +namespace TDengineDriver +{ + class TDengineTest + { + //connect parameters + private string host; + private string configDir; + private string user; + private string password; + private short port = 0; + + //sql parameters + private string dbName = "db"; + private string stableName = "st"; + private string tablePrefix = "t"; + + private bool isInsertOnly = false; + private int queryMode = 1; + + private long recordsPerTable = 10000; + private int recordsPerRequest = 1; + private int colsPerRecord = 3; + private long batchRows = 1000; + private long numOfTables = 10000; + + private IntPtr conn = IntPtr.Zero; + // private long rowsInserted = 0; + private bool useStable = false; + private short methodOfDelete = 0; + private long numOfThreads = 1; + private long rateOfOutorder = 0; + private bool order = true; + private bool skipReadKey = false; + private bool verbose = false; + + + static void PrintHelp(String[] argv) + { + for (int i = 0; i < argv.Length; ++i) + { + if ("--help" == argv[i]) + { + Console.WriteLine("Usage: mono taosdemo.exe [OPTION...]"); + Console.WriteLine(""); + string indent = " "; + Console.Write("{0}{1}", indent, "-h"); + Console.Write("{0}{1}{2}\n", indent, indent, "host, The host to connect to TDengine. Default is localhost."); + Console.Write("{0}{1}", indent, "-p"); + Console.Write("{0}{1}{2}\n", indent, indent, "port, The TCP/IP port number to use for the connection. Default is 0."); + Console.Write("{0}{1}", indent, "-u"); + Console.Write("{0}{1}{2}\n", indent, indent, "user, The user name to use when connecting to the server. Default is 'root'."); + Console.Write("{0}{1}", indent, "-P"); + Console.Write("{0}{1}{2}\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); + Console.Write("{0}{1}", indent, "-d"); + Console.Write("{0}{1}{2}\n", indent, indent, "database, Destination database. Default is 'test'."); + Console.Write("{0}{1}", indent, "-a"); + Console.Write("{0}{1}{2}\n", indent, indent, "replica, Set the replica parameters of the database, Default 1, min: 1, max: 3."); + Console.Write("{0}{1}", indent, "-m"); + Console.Write("{0}{1}{2}\n", indent, indent, "table_prefix, Table prefix name. Default is 't'."); + Console.Write("{0}{1}", indent, "-s"); + Console.Write("{0}{1}{2}\n", indent, indent, "sql file, The select sql file."); + Console.Write("{0}{1}", indent, "-M"); + Console.Write("{0}{1}{2}\n", indent, indent, "stable, Use super table."); + Console.Write("{0}{1}", indent, "-o"); + Console.Write("{0}{1}{2}\n", indent, indent, "outputfile, Direct output to the named file. Default is './output.txt'."); + Console.Write("{0}{1}", indent, "-q"); + Console.Write("{0}{1}{2}\n", indent, indent, "query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + Console.Write("{0}{1}", indent, "-b"); + Console.Write("{0}{1}{2}\n", indent, indent, "type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'."); + Console.Write("{0}{1}", indent, "-w"); + Console.Write("{0}{1}{2}\n", indent, indent, "length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8"); + Console.Write("{0}{1}", indent, "-l"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_cols_per_record, The number of columns per record. Default is 3."); + Console.Write("{0}{1}", indent, "-T"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_threads, The number of threads. Default is 10."); + Console.Write("{0}{1}", indent, "-r"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_records_per_req, The number of records per request. Default is 1000."); + Console.Write("{0}{1}", indent, "-t"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_tables, The number of tables. Default is 10000."); + Console.Write("{0}{1}", indent, "-n"); + Console.Write("{0}{1}{2}\n", indent, indent, "num_of_records_per_table, The number of records per table. Default is 10000."); + Console.Write("{0}{1}", indent, "-c"); + Console.Write("{0}{1}{2}\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); + Console.Write("{0}{1}", indent, "-x"); + Console.Write("{0}{1}{2}\n", indent, indent, "flag, Insert only flag."); + Console.Write("{0}{1}", indent, "-O"); + Console.Write("{0}{1}{2}\n", indent, indent, "order, Insert mode--0: In order, 1: Out of order. Default is in order."); + Console.Write("{0}{1}", indent, "-R"); + Console.Write("{0}{1}{2}\n", indent, indent, "rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50."); + Console.Write("{0}{1}", indent, "-D"); + Console.Write("{0}{1}{2}\n", indent, indent, "Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database."); + Console.Write("{0}{1}", indent, "-v"); + Console.Write("{0}{1}{2}\n", indent, indent, "Print verbose output"); + Console.Write("{0}{1}", indent, "-y"); + Console.Write("{0}{1}{2}\n", indent, indent, "Skip read key for continous test, default is not skip"); + + System.Environment.Exit(0); + } + } + } + + public void ReadArgument(String[] argv) + { + host = this.GetArgumentAsString(argv, "-h", "127.0.0.1"); + port = (short)this.GetArgumentAsLong(argv, "-p", 0, 65535, 6030); + user = this.GetArgumentAsString(argv, "-u", "root"); + password = this.GetArgumentAsString(argv, "-P", "taosdata"); + dbName = this.GetArgumentAsString(argv, "-d", "db"); + stableName = this.GetArgumentAsString(argv, "-s", "st"); + tablePrefix = this.GetArgumentAsString(argv, "-m", "t"); + isInsertOnly = this.GetArgumentAsFlag(argv, "-x"); + queryMode = (int)this.GetArgumentAsLong(argv, "-q", 0, 1, 0); + numOfTables = this.GetArgumentAsLong(argv, "-t", 1, 1000000000, 10000); + batchRows = this.GetArgumentAsLong(argv, "-r", 1, 10000, 1000); + recordsPerTable = this.GetArgumentAsLong(argv, "-n", 1, 100000000000, 10000); + recordsPerRequest = (int)this.GetArgumentAsLong(argv, "-r", 1, 10000, 1); + colsPerRecord = (int)this.GetArgumentAsLong(argv, "-l", 1, 1024, 3); + configDir = this.GetArgumentAsString(argv, "-c", "C:/TDengine/cfg"); + useStable = this.GetArgumentAsFlag(argv, "-M"); + + methodOfDelete = (short)this.GetArgumentAsLong(argv, "-D", 0, 3, 0); + numOfThreads = (short)this.GetArgumentAsLong(argv, "-T", 1, 10000, 1); + order = this.GetArgumentAsFlag(argv, "-O"); + rateOfOutorder = this.GetArgumentAsLong(argv, "-R", 0, 100, 0); + + skipReadKey = this.GetArgumentAsFlag(argv, "-y"); + verbose = this.GetArgumentAsFlag(argv, "-v"); + + Console.Write("###################################################################\n"); + Console.Write("# Server IP: {0}\n", host); + Console.Write("# User: {0}\n", user); + Console.Write("# Password: {0}\n", password); + Console.Write("# Use super table: {0}\n", useStable); + Console.Write("# Number of Columns per record: {0}\n", colsPerRecord); + Console.Write("# Number of Threads: {0}\n", numOfThreads); + Console.Write("# Number of Tables: {0}\n", numOfTables); + Console.Write("# Number of Data per Table: {0}\n", recordsPerTable); + Console.Write("# Records/Request: {0}\n", recordsPerRequest); + Console.Write("# Database name: {0}\n", dbName); + Console.Write("# Table prefix: {0}\n", tablePrefix); + Console.Write("# Data order: {0}\n", order); + Console.Write("# Data out of order rate: {0}\n", rateOfOutorder); + Console.Write("# Delete method: {0}\n", methodOfDelete); + Console.Write("# Query Mode: {0}\n", queryMode); + Console.Write("# Insert Only: {0}\n", isInsertOnly); + Console.Write("# Verbose output {0}\n", verbose); + Console.Write("# Test time: {0}\n", DateTime.Now.ToString("h:mm:ss tt")); + + Console.Write("###################################################################\n"); + + if (skipReadKey == false) + { + Console.Write("Press any key to continue..\n"); + Console.ReadKey(); + } + } + + public bool GetArgumentAsFlag(String[] argv, String argName) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName == argv[i]) + { + return true; + } + } + return false; + } + + public long GetArgumentAsLong(String[] argv, String argName, int minVal, long maxVal, int defaultValue) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName != argv[i]) + { + continue; + } + if (i < argc - 1) + { + String tmp = argv[i + 1]; + if (tmp[0] == '-') + { + Console.WriteLine("option {0:G} requires an argument", tmp); + ExitProgram(); + } + + long tmpVal = Convert.ToInt64(tmp); + if (tmpVal < minVal || tmpVal > maxVal) + { + Console.WriteLine("option {0:G} should in range [{1:G}, {2:G}]", argName, minVal, maxVal); + ExitProgram(); + } + + return tmpVal; + } + } + + return defaultValue; + } + + public String GetArgumentAsString(String[] argv, String argName, String defaultValue) + { + int argc = argv.Length; + for (int i = 0; i < argc; ++i) + { + if (argName != argv[i]) + { + continue; + } + if (i < argc - 1) + { + String tmp = argv[i + 1]; + if (tmp[0] == '-') + { + Console.WriteLine("option {0:G} requires an argument", tmp); + ExitProgram(); + } + return tmp; + } + } + + return defaultValue; + } + + static void ExitProgram() + { + TDengine.Cleanup(); + System.Environment.Exit(0); + } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + public void InitTDengine() + { + TDengine.Options((int)TDengineInitOption.TDDB_OPTION_CONFIGDIR, this.configDir); + TDengine.Options((int)TDengineInitOption.TDDB_OPTION_SHELL_ACTIVITY_TIMER, "60"); + TDengine.Init(); + DebugPrint("TDengine Initialization finished\n"); + } + + public void ConnectTDengine() + { + string db = ""; + DebugPrintFormat("host:{0} user:{1}, pass:{2}; db:{3}, port:{4}", + this.host, this.user, this.password, db, this.port); + this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); + if (this.conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + ExitProgram(); + } + else + { + DebugPrint("Connect to TDengine success\n"); + } + } + + public void CreateTablesByThreads() + { + Thread[] threadArr = new Thread[numOfThreads]; + + long quotition = numOfTables / numOfThreads; + if (quotition < 1) + { + numOfThreads = numOfTables; + quotition = 1; + } + + long remainder = 0; + if (numOfThreads != 0) + { + remainder = numOfTables % numOfThreads; + } + + long last = 0; + + for (int i = 0; i < numOfThreads; i++) + { + CreateTableThread createTableThread = new CreateTableThread(); + createTableThread.id = i; + createTableThread.verbose = verbose; + createTableThread.dbName = this.dbName; + createTableThread.tablePrefix = this.tablePrefix; + if (useStable) + { + createTableThread.stableName = stableName; + } + createTableThread.conn = conn; + + createTableThread.start = last; + if (i < remainder) + { + createTableThread.end = last + quotition; + } + else + { + createTableThread.end = last + quotition - 1; + } + last = createTableThread.end + 1; + + threadArr[i] = new Thread(createTableThread.ThreadMain); + threadArr[i].Start(); + threadArr[i].Join(); + } + } + + public void dropDatabase() + { + StringBuilder sql = new StringBuilder(); + sql.Append("DROP DATABASE IF EXISTS ").Append(this.dbName); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + + } + + public void CreateDb() + { + StringBuilder sql = new StringBuilder(); + sql.Append("CREATE DATABASE IF NOT EXISTS ").Append(this.dbName); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + TDengine.FreeResult(res); + } + + public void CreateStable() + { + StringBuilder sql = new StringBuilder(); + + sql.Clear(); + sql.Append("CREATE TABLE IF NOT EXISTS "). + Append(this.dbName).Append(".").Append(this.stableName). + Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + TDengine.FreeResult(res); + } + + public void InsertByThreads() + { + Thread[] threadArr = new Thread[numOfThreads]; + + long quotition = numOfTables / numOfThreads; + if (quotition < 1) + { + numOfThreads = numOfTables; + quotition = 1; + } + + long remainder = 0; + if (numOfThreads != 0) + { + remainder = numOfTables % numOfThreads; + } + + long last = 0; + + for (int i = 0; i < numOfThreads; i++) + { + InsertDataThread insertThread = new InsertDataThread(); + insertThread.id = i; + insertThread.recordsPerTable = recordsPerTable; + insertThread.batchRows = batchRows; + insertThread.numOfTables = numOfTables; + insertThread.verbose = verbose; + insertThread.dbName = this.dbName; + insertThread.tablePrefix = this.tablePrefix; + if (useStable) + { + insertThread.stableName = stableName; + } + insertThread.conn = conn; + + insertThread.start = last; + if (i < remainder) + { + insertThread.end = last + quotition; + } + else + { + insertThread.end = last + quotition - 1; + } + last = insertThread.end + 1; + + threadArr[i] = new Thread(insertThread.ThreadMain); + threadArr[i].Start(); + threadArr[i].Join(); + } + } + + public void ExecuteQuery() + { + // System.DateTime start = new System.DateTime(); + long queryRows = 0; + + for (int i = 0; i < 1/*this.numOfTables*/; ++i) + { + String sql = "select * from " + this.dbName + "." + tablePrefix + i; + // Console.WriteLine(sql); + + IntPtr res = TDengine.Query(conn, sql); + if (res == IntPtr.Zero) + { + Console.WriteLine(sql + " failure, reason: " + TDengine.Error(res)); + ExitProgram(); + } + + int fieldCount = TDengine.FieldCount(res); + // Console.WriteLine("field count: " + fieldCount); + + List metas = TDengine.FetchFields(res); + for (int j = 0; j < metas.Count; j++) + { + TDengineMeta meta = (TDengineMeta)metas[j]; + // Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); + } + + IntPtr rowdata; + StringBuilder builder = new StringBuilder(); + while ((rowdata = TDengine.FetchRows(res)) != IntPtr.Zero) + { + queryRows++; + for (int fields = 0; fields < fieldCount; ++fields) + { + TDengineMeta meta = metas[fields]; + int offset = IntPtr.Size * fields; + IntPtr data = Marshal.ReadIntPtr(rowdata, offset); + + builder.Append("---"); + + if (data == IntPtr.Zero) + { + builder.Append("NULL"); + continue; + } + + switch ((TDengineDataType)meta.type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + bool v1 = Marshal.ReadByte(data) == 0 ? false : true; + builder.Append(v1); + break; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + byte v2 = Marshal.ReadByte(data); + builder.Append(v2); + break; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + short v3 = Marshal.ReadInt16(data); + builder.Append(v3); + break; + case TDengineDataType.TSDB_DATA_TYPE_INT: + int v4 = Marshal.ReadInt32(data); + builder.Append(v4); + break; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + long v5 = Marshal.ReadInt64(data); + builder.Append(v5); + break; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); + builder.Append(v6); + break; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); + builder.Append(v7); + break; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + string v8 = Marshal.PtrToStringAnsi(data); + builder.Append(v8); + break; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + long v9 = Marshal.ReadInt64(data); + builder.Append(v9); + break; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + string v10 = Marshal.PtrToStringAnsi(data); + builder.Append(v10); + break; + } + } + builder.Append("---"); + + if (queryRows <= 10) + { + Console.WriteLine(builder.ToString()); + } + builder.Clear(); + } + + if (TDengine.ErrorNo(res) != 0) + { + Console.Write("Query is not complete, Error {0:G}", + TDengine.ErrorNo(res), TDengine.Error(res)); + } + + TDengine.FreeResult(res); + } + /* + System.DateTime end = new System.DateTime(); + TimeSpan ts = end - start; + + Console.Write("Total {0:G} rows inserted, {1:G} rows query, time spend {2:G} seconds.\n" + , this.rowsInserted, queryRows, ts.TotalSeconds); + */ + } + + public void CloseConnection() + { + if (this.conn != IntPtr.Zero) + { + TDengine.Close(this.conn); + } + } + + // Main entry + static void Main(string[] args) + { + PrintHelp(args); + + TDengineTest tester = new TDengineTest(); + tester.ReadArgument(args); + + tester.InitTDengine(); + tester.ConnectTDengine(); + tester.dropDatabase(); + tester.CreateDb(); + + if (tester.useStable == true) + { + tester.CreateStable(); + } + + tester.CreateTablesByThreads(); + + Stopwatch watch = Stopwatch.StartNew(); + tester.InsertByThreads(); + watch.Stop(); + double elapsedMs = watch.Elapsed.TotalMilliseconds; + + Console.WriteLine("Spent {0} seconds to insert {1} records with {2} record(s) per request: {3} records/second", + elapsedMs / 1000, + tester.recordsPerTable * tester.numOfTables, + tester.batchRows, + (tester.recordsPerTable * tester.numOfTables * 1000) / elapsedMs); + + tester.ExecuteQuery(); + tester.CloseConnection(); + + Console.WriteLine("End."); + } + + public class InsertDataThread + { + public long id { set; get; } + public long start { set; get; } + public long end { set; get; } + public string dbName { set; get; } + public IntPtr conn { set; get; } + public string tablePrefix { set; get; } + public string stableName { set; get; } + public long recordsPerTable { set; get; } + public long batchRows { set; get; } + public long numOfTables { set; get; } + public bool verbose { set; get; } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + public void ThreadMain() + { + DebugPrintFormat("InsertDataThread {0} from {1} to {2}", id, start, end); + StringBuilder sql = new StringBuilder(); + long beginTimestamp = 1551369600000L; + long rowsInserted = 0; + + // System.DateTime startTime = new System.DateTime(); + long i = 0; + while (i < recordsPerTable) + { + for (long table = start; table <= end; ++table) + { + long inserted = i; + + sql.Clear(); + sql.Append("INSERT INTO "). + Append(this.dbName).Append(".").Append(this.tablePrefix).Append(table). + Append(" VALUES"); + for (int batch = 0; batch < this.batchRows; ++batch) + { + sql.Append("(") + .Append(beginTimestamp + i + batch) + .Append(", 1, 2, 3,") + .Append(i + batch) + .Append(", 5, 6, 7, 'abc', 'def')"); + + } + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res == IntPtr.Zero) + { + DebugPrint(sql.ToString() + " failure, reason: " + TDengine.Error(res) + "\n"); + } + + inserted += this.batchRows; + + int affectRows = TDengine.AffectRows(res); + rowsInserted += affectRows; + + TDengine.FreeResult(res); + if (table == end) + { + i = inserted; + } + } + } + + } + } + + public class CreateTableThread + { + public long id { set; get; } + public long start { set; get; } + public long end { set; get; } + public string dbName { set; get; } + public IntPtr conn { set; get; } + public string tablePrefix { set; get; } + public string stableName { set; get; } + public bool verbose { set; get; } + + private void DebugPrintFormat(string format, params object[] parameters) + { + if (verbose == true) + { + Console.Write(format, parameters); + } + } + + private void DebugPrint(string str) + { + if (verbose == true) + { + Console.Write(str); + } + } + + public void ThreadMain() + { + DebugPrintFormat("CreateTable {0} from {1} to {2}", id, start, end); + + StringBuilder sql = new StringBuilder(); + + for (long tableId = start; tableId <= end; tableId++) + { + sql.Clear(); + sql = sql.Append("CREATE TABLE IF NOT EXISTS "). + Append(this.dbName).Append(".").Append(this.tablePrefix).Append(tableId). + Append(" USING ").Append(this.dbName).Append(".").Append(this.stableName). + Append(" TAGS(").Append(tableId).Append(")"); + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) + { + DebugPrint(sql.ToString() + " success\n"); + } + else + { + DebugPrint(sql.ToString() + " failure, reason: " + TDengine.Error(res) + "\n"); + ExitProgram(); + } + TDengine.FreeResult(res); + } + + } + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index 98f908b77e8d3bfa03eebf8560205b0424720f1f..2a862dad6e84c8d22dc615de7b75f8e5fe3823b5 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -10,12 +10,6 @@ jar - - org.apache.maven.plugins - maven-assembly-plugin - 3.0.0 - - org.apache.maven.plugins maven-assembly-plugin diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java index 259985ec9f4708b9317575fd97919adcc82d7161..cbf63b028e78d57165ecb1890270cc7e87239f24 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/JdbcTaosdemo.java @@ -29,11 +29,7 @@ public class JdbcTaosdemo { JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); boolean isHelp = Arrays.asList(args).contains("--help"); - if (isHelp) { - JdbcTaosdemoConfig.printHelp(); - return; - } - if (config.getHost() == null) { + if (isHelp || config.host == null || config.host.isEmpty()) { JdbcTaosdemoConfig.printHelp(); return; } @@ -85,7 +81,7 @@ public class JdbcTaosdemo { taosdemo.selectLastOneYear(); // drop super table - if (config.isDeleteTable()) + if (config.dropTable) taosdemo.dropSuperTable(); taosdemo.close(); } @@ -103,7 +99,7 @@ public class JdbcTaosdemo { logger.info("[ OK ] Connection established."); } catch (ClassNotFoundException | SQLException e) { logger.error(e.getMessage()); - throw new RuntimeException("connection failed: " + config.getHost()); + throw new RuntimeException("connection failed: " + config.host); } } @@ -111,7 +107,7 @@ public class JdbcTaosdemo { * create database */ private void createDatabase() { - String sql = SqlSpeller.createDatabaseSQL(config.getDbName(), config.getKeep(), config.getDays()); + String sql = SqlSpeller.createDatabaseSQL(config.database, config.keep, config.days); execute(sql); } @@ -119,7 +115,7 @@ public class JdbcTaosdemo { * drop database */ private void dropDatabase() { - String sql = SqlSpeller.dropDatabaseSQL(config.getDbName()); + String sql = SqlSpeller.dropDatabaseSQL(config.database); execute(sql); } @@ -127,7 +123,7 @@ public class JdbcTaosdemo { * use database */ private void useDatabase() { - String sql = SqlSpeller.useDatabaseSQL(config.getDbName()); + String sql = SqlSpeller.useDatabaseSQL(config.database); execute(sql); } @@ -135,7 +131,7 @@ public class JdbcTaosdemo { * create super table */ private void createSuperTable() { - String sql = SqlSpeller.createSuperTableSQL(config.getStbName()); + String sql = SqlSpeller.createSuperTableSQL(config.superTable); execute(sql); } @@ -144,9 +140,9 @@ public class JdbcTaosdemo { */ private void createTableMultiThreads() { try { - final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); + final int tableSize = (int) (config.numOfTables / config.numOfThreadsForCreate); List threads = new ArrayList<>(); - for (int i = 0; i < config.getNumberOfThreads(); i++) { + for (int i = 0; i < config.numOfThreadsForCreate; i++) { Thread thread = new Thread(new CreateTableTask(config, i * tableSize, tableSize), "Thread-" + i); threads.add(thread); thread.start(); @@ -169,9 +165,9 @@ public class JdbcTaosdemo { final long startDatetime = TimeStampUtil.datetimeToLong("2005-01-01 00:00:00.000"); final long finishDatetime = TimeStampUtil.datetimeToLong("2030-01-01 00:00:00.000"); - final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); + final int tableSize = (int) (config.numOfTables / config.numOfThreadsForInsert); List threads = new ArrayList<>(); - for (int i = 0; i < config.getNumberOfThreads(); i++) { + for (int i = 0; i < config.numOfThreadsForInsert; i++) { Thread thread = new Thread(new InsertTableDatetimeTask(config, i * tableSize, tableSize, startDatetime, finishDatetime), "Thread-" + i); threads.add(thread); thread.start(); @@ -188,10 +184,10 @@ public class JdbcTaosdemo { private void insertMultiThreads() { try { - final int tableSize = config.getNumberOfTable() / config.getNumberOfThreads(); - final int numberOfRecordsPerTable = config.getNumberOfRecordsPerTable(); + final int tableSize = (int) (config.numOfTables / config.numOfThreadsForInsert); + final int numberOfRecordsPerTable = (int) config.numOfRowsPerTable; List threads = new ArrayList<>(); - for (int i = 0; i < config.getNumberOfThreads(); i++) { + for (int i = 0; i < config.numOfThreadsForInsert; i++) { Thread thread = new Thread(new InsertTableTask(config, i * tableSize, tableSize, numberOfRecordsPerTable), "Thread-" + i); threads.add(thread); thread.start(); @@ -207,86 +203,85 @@ public class JdbcTaosdemo { } private void selectFromTableLimit() { - String sql = SqlSpeller.selectFromTableLimitSQL(config.getDbName(), config.getTbPrefix(), 1, 10, 0); + String sql = SqlSpeller.selectFromTableLimitSQL(config.database, config.prefixOfTable, 1, 10, 0); executeQuery(sql); } private void selectCountFromTable() { - String sql = SqlSpeller.selectCountFromTableSQL(config.getDbName(), config.getTbPrefix(), 1); + String sql = SqlSpeller.selectCountFromTableSQL(config.database, config.prefixOfTable, 1); executeQuery(sql); } private void selectAvgMinMaxFromTable() { - String sql = SqlSpeller.selectAvgMinMaxFromTableSQL("current", config.getDbName(), config.getTbPrefix(), 1); + String sql = SqlSpeller.selectAvgMinMaxFromTableSQL("current", config.database, config.prefixOfTable, 1); executeQuery(sql); } private void selectLastFromTable() { - String sql = SqlSpeller.selectLastFromTableSQL(config.getDbName(), config.getTbPrefix(), 1); + String sql = SqlSpeller.selectLastFromTableSQL(config.database, config.prefixOfTable, 1); executeQuery(sql); } private void selectFromSuperTableLimit() { - String sql = SqlSpeller.selectFromSuperTableLimitSQL(config.getDbName(), config.getStbName(), 10, 0); + String sql = SqlSpeller.selectFromSuperTableLimitSQL(config.database, config.superTable, 10, 0); executeQuery(sql); } private void selectCountFromSuperTable() { - String sql = SqlSpeller.selectCountFromSuperTableSQL(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectCountFromSuperTableSQL(config.database, config.superTable); executeQuery(sql); } private void selectAvgMinMaxFromSuperTable() { - String sql = SqlSpeller.selectAvgMinMaxFromSuperTableSQL("current", config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectAvgMinMaxFromSuperTableSQL("current", config.database, config.superTable); executeQuery(sql); } private void selectAvgMinMaxFromSuperTableWhereTag() { - String sql = SqlSpeller.selectAvgMinMaxFromSuperTableWhere("current", config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectAvgMinMaxFromSuperTableWhere("current", config.database, config.superTable); executeQuery(sql); } private void selectLastFromSuperTableWhere() { - String sql = SqlSpeller.selectLastFromSuperTableWhere("current", config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectLastFromSuperTableWhere("current", config.database, config.superTable); executeQuery(sql); } private void selectGroupBy() { - String sql = SqlSpeller.selectGroupBy("current", config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectGroupBy("current", config.database, config.superTable); executeQuery(sql); } private void selectLike() { - String sql = SqlSpeller.selectLike(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectLike(config.database, config.superTable); executeQuery(sql); } private void selectLastOneHour() { - String sql = SqlSpeller.selectLastOneHour(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectLastOneHour(config.database, config.superTable); executeQuery(sql); } private void selectLastOneDay() { - String sql = SqlSpeller.selectLastOneDay(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectLastOneDay(config.database, config.superTable); executeQuery(sql); } private void selectLastOneWeek() { - String sql = SqlSpeller.selectLastOneWeek(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectLastOneWeek(config.database, config.superTable); executeQuery(sql); } private void selectLastOneMonth() { - String sql = SqlSpeller.selectLastOneMonth(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectLastOneMonth(config.database, config.superTable); executeQuery(sql); } private void selectLastOneYear() { - String sql = SqlSpeller.selectLastOneYear(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.selectLastOneYear(config.database, config.superTable); executeQuery(sql); } - private void close() { try { if (connection != null) { @@ -303,7 +298,7 @@ public class JdbcTaosdemo { * drop super table */ private void dropSuperTable() { - String sql = SqlSpeller.dropSuperTableSQL(config.getDbName(), config.getStbName()); + String sql = SqlSpeller.dropSuperTableSQL(config.database, config.superTable); execute(sql); } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java index 36745a93941cc690f37d06d9a3662605723bbd2c..e374f3a39f895701655e4056e5399d4dfdcd1276 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java @@ -1,54 +1,90 @@ package com.taosdata.example.jdbcTaosdemo.domain; -public final class JdbcTaosdemoConfig { - - //The host to connect to TDengine. Must insert one - private String host; - //The TCP/IP port number to use for the connection. Default is 6030. - private int port = 6030; - //The TDengine user name to use when connecting to the server. Default is 'root' - private String user = "root"; - //The password to use when connecting to the server. Default is 'taosdata' - private String password = "taosdata"; - - //Destination database. Default is 'test' - private String dbName = "test"; - //keep - private int keep = 36500; - //days - private int days = 120; - - //Super table Name. Default is 'meters' - private String stbName = "meters"; - //Table name prefix. Default is 'd' - private String tbPrefix = "d"; - //The number of tables. Default is 10. - private int numberOfTable = 10; - //The number of records per table. Default is 2 - private int numberOfRecordsPerTable = 2; - //The number of records per request. Default is 100 - private int numberOfRecordsPerRequest = 100; +import com.taosdata.example.jdbcTaosdemo.utils.TimeStampUtil; - //The number of threads. Default is 1. - private int numberOfThreads = 1; - //Delete data. Default is false - private boolean deleteTable = false; +public final class JdbcTaosdemoConfig { + // instance + public String host; //host + public int port = 6030; //port + public String user = "root"; //user + public String password = "taosdata"; //password + // database + public String database = "test"; //database + public int keep = 3650; //keep + public int days = 30; //days + public int replica = 1; //replica + //super table + public boolean doCreateTable = true; + public String superTable = "weather"; //super table name + public String prefixOfFields = "col"; + public int numOfFields; + public String prefixOfTags = "tag"; + public int numOfTags; + public String superTableSQL; + //sub table + public String prefixOfTable = "t"; + // insert task + public boolean autoCreateTable = true; + public long numOfTables = 100; + public long numOfRowsPerTable = 100; + public int numOfTablesPerSQL = 10; + public int numOfValuesPerSQL = 10; + public int numOfThreadsForCreate = 1; + public int numOfThreadsForInsert = 1; + public long startTime; + public long timeGap = 1; + public int frequency; + public int order; + public int rate = 10; + public long range = 1000l; + // select task + + // drop task + public boolean dropTable = false; public static void printHelp() { - System.out.println("Usage: java -jar JdbcTaosDemo.jar [OPTION...]"); - System.out.println("-h host The host to connect to TDengine. you must input one"); - System.out.println("-p port The TCP/IP port number to use for the connection. Default is 6030"); - System.out.println("-u user The TDengine user name to use when connecting to the server. Default is 'root'"); - System.out.println("-P password The password to use when connecting to the server.Default is 'taosdata'"); - System.out.println("-d database Destination database. Default is 'test'"); - System.out.println("-m tablePrefix Table prefix name. Default is 'd'"); - System.out.println("-t num_of_tables The number of tables. Default is 10"); - System.out.println("-n num_of_records_per_table The number of records per table. Default is 2"); - System.out.println("-r num_of_records_per_req The number of records per request. Default is 100"); - System.out.println("-T num_of_threads The number of threads. Default is 1"); - System.out.println("-D delete table Delete data methods. Default is false"); - System.out.println("--help Give this help list"); -// System.out.println("--infinite infinite insert mode"); + System.out.println("Usage: java -jar jdbc-taosdemo-2.0.jar [OPTION...]"); + // instance + System.out.println("-host The host to connect to TDengine which you must specify"); + System.out.println("-port The TCP/IP port number to use for the connection. Default is 6030"); + System.out.println("-user The TDengine user name to use when connecting to the server. Default is 'root'"); + System.out.println("-password The password to use when connecting to the server.Default is 'taosdata'"); + // database + System.out.println("-database Destination database. Default is 'test'"); + System.out.println("-keep database keep parameter. Default is 3650"); + System.out.println("-days database days parameter. Default is 30"); + System.out.println("-replica database replica parameter. Default 1, min: 1, max: 3"); + // super table + System.out.println("-doCreateTable do create super table and sub table, true or false, Default true"); + System.out.println("-superTable super table name. Default 'weather'"); + System.out.println("-prefixOfFields The prefix of field in super table. Default is 'col'"); + System.out.println("-numOfFields The number of field in super table. Default is (ts timestamp, temperature float, humidity int)."); + System.out.println("-prefixOfTags The prefix of tag in super table. Default is 'tag'"); + System.out.println("-numOfTags The number of tag in super table. Default is (location nchar(64), groupId int)."); + System.out.println("-superTableSQL specify a sql statement for the super table.\n" + + " Default is 'create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int). \n" + + " if you use this parameter, the numOfFields and numOfTags will be invalid'"); + // sub table + System.out.println("-prefixOfTable The prefix of sub tables. Default is 't'"); + System.out.println("-numOfTables The number of tables. Default is 1"); + System.out.println("-numOfThreadsForCreate The number of thread during create sub table. Default is 1"); + // insert task + System.out.println("-autoCreateTable Use auto Create sub tables SQL. Default is false"); + System.out.println("-numOfRowsPerTable The number of records per table. Default is 1"); + System.out.println("-numOfThreadsForInsert The number of threads during insert row. Default is 1"); + System.out.println("-numOfTablesPerSQL The number of table per SQL. Default is 1"); + System.out.println("-numOfValuesPerSQL The number of value per SQL. Default is 1"); + System.out.println("-startTime start time for insert task, The format is \"yyyy-MM-dd HH:mm:ss.SSS\"."); + System.out.println("-timeGap the number of time gap. Default is 1000 ms"); + System.out.println("-frequency the number of records per second inserted into one table. default is 0, do not control frequency"); + System.out.println("-order Insert mode--0: In order, 1: Out of order. Default is in order"); + System.out.println("-rate The proportion of data out of order. effective only if order is 1. min 0, max 100, default is 10"); + System.out.println("-range The range of data out of order. effective only if order is 1. default is 1000 ms"); + // query task +// System.out.println("-sqlFile The select sql file"); + // drop task + System.out.println("-dropTable Drop data before quit. Default is false"); + System.out.println("--help Give this help list"); } /** @@ -59,95 +95,111 @@ public final class JdbcTaosdemoConfig { */ public JdbcTaosdemoConfig(String[] args) { for (int i = 0; i < args.length; i++) { - if ("-h".equals(args[i]) && i < args.length - 1) { + // instance + if ("-host".equals(args[i]) && i < args.length - 1) { host = args[++i]; } - if ("-p".equals(args[i]) && i < args.length - 1) { + if ("-port".equals(args[i]) && i < args.length - 1) { port = Integer.parseInt(args[++i]); } - if ("-u".equals(args[i]) && i < args.length - 1) { + if ("-user".equals(args[i]) && i < args.length - 1) { user = args[++i]; } - if ("-P".equals(args[i]) && i < args.length - 1) { + if ("-password".equals(args[i]) && i < args.length - 1) { password = args[++i]; } - if ("-d".equals(args[i]) && i < args.length - 1) { - dbName = args[++i]; + // database + if ("-database".equals(args[i]) && i < args.length - 1) { + database = args[++i]; } - if ("-m".equals(args[i]) && i < args.length - 1) { - tbPrefix = args[++i]; + if ("-keep".equals(args[i]) && i < args.length - 1) { + keep = Integer.parseInt(args[++i]); } - if ("-t".equals(args[i]) && i < args.length - 1) { - numberOfTable = Integer.parseInt(args[++i]); + if ("-days".equals(args[i]) && i < args.length - 1) { + days = Integer.parseInt(args[++i]); } - if ("-n".equals(args[i]) && i < args.length - 1) { - numberOfRecordsPerTable = Integer.parseInt(args[++i]); + if ("-replica".equals(args[i]) && i < args.length - 1) { + replica = Integer.parseInt(args[++i]); } - if ("-r".equals(args[i]) && i < args.length - 1) { - numberOfRecordsPerRequest = Integer.parseInt(args[++i]); + // super table + if ("-doCreateTable".equals(args[i]) && i < args.length - 1) { + doCreateTable = Boolean.parseBoolean(args[++i]); } - if ("-T".equals(args[i]) && i < args.length - 1) { - numberOfThreads = Integer.parseInt(args[++i]); + if ("-superTable".equals(args[i]) && i < args.length - 1) { + superTable = args[++i]; } - if ("-D".equals(args[i]) && i < args.length - 1) { - deleteTable = Boolean.parseBoolean(args[++i]); + if ("-prefixOfFields".equals(args[i]) && i < args.length - 1) { + prefixOfFields = args[++i]; } - } - } - - public String getHost() { - return host; - } - - public int getPort() { - return port; - } - - public String getUser() { - return user; - } - - public String getPassword() { - return password; - } - - public String getDbName() { - return dbName; - } - - public int getKeep() { - return keep; - } - - public int getDays() { - return days; - } - - public String getStbName() { - return stbName; - } - - public String getTbPrefix() { - return tbPrefix; - } - - public int getNumberOfTable() { - return numberOfTable; - } - - public int getNumberOfRecordsPerTable() { - return numberOfRecordsPerTable; - } + if ("-numOfFields".equals(args[i]) && i < args.length - 1) { + numOfFields = Integer.parseInt(args[++i]); + } + if ("-prefixOfTags".equals(args[i]) && i < args.length - 1) { + prefixOfTags = args[++i]; + } + if ("-numOfTags".equals(args[i]) && i < args.length - 1) { + numOfTags = Integer.parseInt(args[++i]); + } + if ("-superTableSQL".equals(args[i]) && i < args.length - 1) { + superTableSQL = args[++i]; + } + // sub table + if ("-prefixOfTable".equals(args[i]) && i < args.length - 1) { + prefixOfTable = args[++i]; + } + if ("-numOfTables".equals(args[i]) && i < args.length - 1) { + numOfTables = Long.parseLong(args[++i]); + } + if ("-autoCreateTable".equals(args[i]) && i < args.length - 1) { + autoCreateTable = Boolean.parseBoolean(args[++i]); + } + if ("-numOfThreadsForCreate".equals(args[i]) && i < args.length - 1) { + numOfThreadsForCreate = Integer.parseInt(args[++i]); + } + // insert task + if ("-numOfRowsPerTable".equals(args[i]) && i < args.length - 1) { + numOfRowsPerTable = Long.parseLong(args[++i]); + } + if ("-numOfThreadsForInsert".equals(args[i]) && i < args.length - 1) { + numOfThreadsForInsert = Integer.parseInt(args[++i]); + } + if ("-numOfTablesPerSQL".equals(args[i]) && i < args.length - 1) { + numOfTablesPerSQL = Integer.parseInt(args[++i]); + } + if ("-numOfValuesPerSQL".equals(args[i]) && i < args.length - 1) { + numOfValuesPerSQL = Integer.parseInt(args[++i]); + } + if ("-startTime".equals(args[i]) && i < args.length - 1) { + startTime = TimeStampUtil.datetimeToLong(args[++i]); + } + if ("-timeGap".equals(args[i]) && i < args.length - 1) { + timeGap = Long.parseLong(args[++i]); + } + if ("-frequency".equals(args[i]) && i < args.length - 1) { + frequency = Integer.parseInt(args[++i]); + } + if ("-order".equals(args[i]) && i < args.length - 1) { + order = Integer.parseInt(args[++i]); + } + if ("-rate".equals(args[i]) && i < args.length - 1) { + rate = Integer.parseInt(args[++i]); + if (rate < 0 || rate > 100) + throw new IllegalArgumentException("rate must between 0 and 100"); + } + if ("-range".equals(args[i]) && i < args.length - 1) { + range = Integer.parseInt(args[++i]); + } + // select task - public int getNumberOfThreads() { - return numberOfThreads; + // drop task + if ("-dropTable".equals(args[i]) && i < args.length - 1) { + dropTable = Boolean.parseBoolean(args[++i]); + } + } } - public boolean isDeleteTable() { - return deleteTable; + public static void main(String[] args) { + JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); } - public int getNumberOfRecordsPerRequest() { - return numberOfRecordsPerRequest; - } } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java index 1da2c8647efe0e9204f1a591ba9431a489c91cb0..b054a008e78dd723e7009c3abfec7bb0f398b4f5 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/CreateTableTask.java @@ -28,7 +28,7 @@ public class CreateTableTask implements Runnable { Connection connection = ConnectionFactory.build(config); for (int i = startIndex; i < startIndex + tableNumber; i++) { Statement statement = connection.createStatement(); - String sql = SqlSpeller.createTableSQL(i + 1, config.getDbName(), config.getStbName()); + String sql = SqlSpeller.createTableSQL(i + 1, config.database, config.superTable); statement.execute(sql); statement.close(); logger.info(">>> " + sql); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java index 4f60c25646573223a9cbfd820c8eb37e4f6f6c8c..fc9275c6f03e2e715c38df94a80141aaf7b80423 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableDatetimeTask.java @@ -30,10 +30,10 @@ public class InsertTableDatetimeTask implements Runnable { public void run() { try { Connection connection = ConnectionFactory.build(config); - int valuesCount = config.getNumberOfRecordsPerRequest(); + int valuesCount = config.numOfValuesPerSQL; for (long ts = startDatetime; ts < finishedDatetime; ts += valuesCount) { for (int i = startTableIndex; i < startTableIndex + tableNumber; i++) { - String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, valuesCount); + String sql = SqlSpeller.insertBatchSizeRowsSQL(config.database, config.prefixOfTable, i + 1, ts, valuesCount); Statement statement = connection.createStatement(); statement.execute(sql); statement.close(); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java index 644de52dd3e75a77c2d635a6f5328f186259096c..733735d780a3af834677256dacd571e4573060ae 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java @@ -31,7 +31,7 @@ public class InsertTableTask implements Runnable { public void run() { try { Connection connection = ConnectionFactory.build(config); - int keep = config.getKeep(); + int keep = config.keep; Instant end = Instant.now(); Instant start = end.minus(Duration.ofDays(keep - 1)); long timeGap = ChronoUnit.MILLIS.between(start, end) / (recordsNumberPerTable - 1); @@ -41,7 +41,7 @@ public class InsertTableTask implements Runnable { long ts = start.toEpochMilli() + (j * timeGap); // insert data into echo table for (int i = startTbIndex; i < startTbIndex + tableNumber; i++) { - String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, config.getNumberOfRecordsPerRequest()); + String sql = SqlSpeller.insertBatchSizeRowsSQL(config.database, config.prefixOfTable, i + 1, ts, config.numOfValuesPerSQL); logger.info(Thread.currentThread().getName() + ">>> " + sql); Statement statement = connection.createStatement(); statement.execute(sql); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java index 52691f4de722db2eb17e7061e09ff0e59a390077..37b46868b6a5c4ee099c0e154740c2d44f6c25e4 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/utils/ConnectionFactory.java @@ -11,7 +11,7 @@ import java.util.Properties; public class ConnectionFactory { public static Connection build(JdbcTaosdemoConfig config) throws SQLException { - return build(config.getHost(), config.getPort(), config.getDbName(), config.getUser(), config.getPassword()); + return build(config.host, config.port, config.database, config.user, config.password); } public static Connection build(String host, int port, String dbName) throws SQLException { diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java index 79c0aacea740dcb6fca9780c7f64872c537c3225..4e33b75bc57aa18ebc0cae166e84401d4357d614 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java @@ -82,12 +82,27 @@ public class ConnectionPoolDemo { init(dataSource); - ExecutorService executor = Executors.newFixedThreadPool(threadCount); - while (true) { - executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); - if (sleep > 0) - TimeUnit.MILLISECONDS.sleep(sleep); + try { + Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement(); + String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; + int affectRows = statement.executeUpdate(sql); + System.out.println("affectRows >>> " + affectRows); + affectRows = statement.executeUpdate(sql); + System.out.println("affectRows >>> " + affectRows); + statement.close(); + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); } + + +// ExecutorService executor = Executors.newFixedThreadPool(threadCount); +// while (true) { +// executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); +// if (sleep > 0) +// TimeUnit.MILLISECONDS.sleep(sleep); +// } } private static void init(DataSource dataSource) { diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java index 69cd3e0ced2888575d890ffea36407455c4bea7a..f693214567fb3e8780e8ee1a867cf765de0ec8d7 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java @@ -6,7 +6,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; @MapperScan(basePackages = {"com.taosdata.jdbc.springbootdemo.dao"}) @SpringBootApplication -public class SpringbootdemoApplication { +public class cd { public static void main(String[] args) { SpringApplication.run(SpringbootdemoApplication.class, args); diff --git a/tests/examples/JDBC/taosdemo/pom.xml b/tests/examples/JDBC/taosdemo/pom.xml index 5cbf6cb7004566e8d6ea6e16e8b70f289ac2941d..15b22917b6103740ac419e046d5fd8fa79363d25 100644 --- a/tests/examples/JDBC/taosdemo/pom.xml +++ b/tests/examples/JDBC/taosdemo/pom.xml @@ -2,88 +2,107 @@ 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 2.4.0 - - com.taosdata taosdemo 2.0 taosdemo + jar Demo project for TDengine - 1.8 + 5.3.2 - - + - com.taosdata.jdbc - taos-jdbcdriver - 2.0.14 + org.springframework + spring-context + ${spring.version} - - mysql - mysql-connector-java - 5.1.47 + org.springframework + spring-core + ${spring.version} - - com.baomidou - mybatis-plus-boot-starter - 3.1.2 + org.springframework + spring-beans + ${spring.version} - - log4j - log4j - 1.2.17 + org.springframework + spring-expression + ${spring.version} + + + org.springframework + spring-aop + ${spring.version} + + + org.springframework + spring-aspects + ${spring.version} + + + org.springframework + spring-test + ${spring.version} + test + + + org.springframework + spring-jdbc + ${spring.version} - + + + com.zaxxer + HikariCP + 3.4.5 + + - org.springframework.boot - spring-boot-starter-jdbc + com.taosdata.jdbc + taos-jdbcdriver + 2.0.15 + system + ${project.basedir}/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar + - org.springframework.boot - spring-boot-starter-thymeleaf + com.alibaba + fastjson + 1.2.75 + - org.springframework.boot - spring-boot-starter-web + mysql + mysql-connector-java + 5.1.47 + test + - org.mybatis.spring.boot - mybatis-spring-boot-starter - 2.1.4 + log4j + log4j + 1.2.17 + junit junit 4.12 test - - org.springframework.boot - spring-boot-devtools - runtime - true - + org.projectlombok lombok - true - - - org.springframework.boot - spring-boot-starter-test - test + 1.18.16 + provided @@ -94,6 +113,7 @@ **/*.properties **/*.xml + **/*.jar true @@ -108,10 +128,41 @@ - org.springframework.boot - spring-boot-maven-plugin + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + + com.taosdata.taosdemo.TaosDemoApplication + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + diff --git a/tests/examples/JDBC/taosdemo/readme.md b/tests/examples/JDBC/taosdemo/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..a4b6e297691c4981fc36a06679d085c183e890a1 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/readme.md @@ -0,0 +1,3 @@ +需求： +1. 可以读lowa的配置文件 +2. 支持对JNI方式和Restful方式的taos-driver \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java new file mode 100644 index 0000000000000000000000000000000000000000..b9a22a1ef75962159bd33a8e525898a69b67a911 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -0,0 +1,112 @@ +package com.taosdata.taosdemo; + +import com.taosdata.taosdemo.components.DataSourceFactory; +import com.taosdata.taosdemo.components.JdbcTaosdemoConfig; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.service.DatabaseService; +import com.taosdata.taosdemo.service.SubTableService; +import com.taosdata.taosdemo.service.SuperTableService; +import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator; +import org.apache.log4j.Logger; + +import javax.sql.DataSource; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class TaosDemoApplication { + + private static Logger logger = Logger.getLogger(TaosDemoApplication.class); + + public static void main(String[] args) throws IOException { + // 读配置参数 + JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); + boolean isHelp = Arrays.asList(args).contains("--help"); + if (isHelp || config.host == null || config.host.isEmpty()) { +// if (isHelp) { + JdbcTaosdemoConfig.printHelp(); + System.exit(0); + } + // 初始化 + final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); + final DatabaseService databaseService = new DatabaseService(dataSource); + final SuperTableService superTableService = new SuperTableService(dataSource); + final SubTableService subTableService = new SubTableService(dataSource); + // 创建数据库 + long start = System.currentTimeMillis(); + Map databaseParam = new HashMap<>(); + databaseParam.put("database", config.database); + databaseParam.put("keep", Integer.toString(config.keep)); + databaseParam.put("days", Integer.toString(config.days)); + databaseParam.put("replica", Integer.toString(config.replica)); + //TODO: other database parameters + databaseService.createDatabase(databaseParam); + databaseService.useDatabase(config.database); + long end = System.currentTimeMillis(); + logger.info(">>> create database time cost : " + (end - start) + " ms."); + /**********************************************************************************/ + // 构造超级表的meta + SuperTableMeta superTableMeta; + // create super table + if (config.superTableSQL != null) { + // use a sql to create super table + superTableMeta = SuperTableMetaGenerator.generate(config.superTableSQL); + if (config.database != null && !config.database.isEmpty()) + superTableMeta.setDatabase(config.database); + } else if (config.numOfFields == 0) { + String sql = "create table " + config.database + "." + config.superTable + " (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; + superTableMeta = SuperTableMetaGenerator.generate(sql); + } else { + // create super table with specified field size and tag size + superTableMeta = SuperTableMetaGenerator.generate(config.database, config.superTable, config.numOfFields, config.prefixOfFields, config.numOfTags, config.prefixOfTags); + } + /**********************************************************************************/ + // 建表 + start = System.currentTimeMillis(); + if (config.doCreateTable) { + superTableService.drop(superTableMeta.getDatabase(), superTableMeta.getName()); + superTableService.create(superTableMeta); + if (!config.autoCreateTable) { + // 批量建子表 + subTableService.createSubTable(superTableMeta, config.numOfTables, config.prefixOfTable, config.numOfThreadsForCreate); + } + } + end = System.currentTimeMillis(); + logger.error(">>> create table time cost : " + (end - start) + " ms."); + /**********************************************************************************/ + // 插入 + long tableSize = config.numOfTables; + int threadSize = config.numOfThreadsForInsert; + long startTime = getProperStartTime(config.startTime, config.keep); + + if (tableSize < threadSize) + threadSize = (int) tableSize; + long gap = (long) Math.ceil((0.0d + tableSize) / threadSize); + + start = System.currentTimeMillis(); + // multi threads to insert + int affectedRows = subTableService.insertMultiThreads(superTableMeta, threadSize, tableSize, startTime, gap, config); + end = System.currentTimeMillis(); + logger.error("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms"); + /**********************************************************************************/ + // 删除表 + if (config.dropTable) { + superTableService.drop(config.database, config.superTable); + } + System.exit(0); + } + + private static long getProperStartTime(long startTime, int keep) { + Instant now = Instant.now(); + long earliest = now.minus(Duration.ofDays(keep - 1)).toEpochMilli(); + if (startTime == 0 || startTime < earliest) { + startTime = earliest; + } + return startTime; + } + + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosdemoApplication.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosdemoApplication.java deleted file mode 100644 index db1b20527d88f5bca24ed5c6a3b477fa3a71bdf1..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosdemoApplication.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.taosdata.taosdemo; - -import org.mybatis.spring.annotation.MapperScan; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -@MapperScan(basePackages = {"com.taosdata.taosdemo.mapper"}) -@SpringBootApplication -public class TaosdemoApplication { - - public static void main(String[] args) { - SpringApplication.run(TaosdemoApplication.class, args); - } - -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..c96d6f8bed68e9bb67d959ddb1d7531b4cbadeb3 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java @@ -0,0 +1,63 @@ +package com.taosdata.taosdemo.components; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.stereotype.Component; + +import javax.sql.DataSource; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +@Component +public class DataSourceFactory { + + private static DataSource instance; + + public static DataSource getInstance(String host, int port, String user, String password) throws IOException { + if (instance == null) { + synchronized (DataSourceFactory.class) { + if (instance == null) { + InputStream is = DataSourceFactory.class.getClassLoader().getResourceAsStream("application.properties"); + Properties properties = new Properties(); + properties.load(is); + + HikariConfig config = new HikariConfig(); + + if (properties.containsKey("jdbc.driver")) { +// String driverName = properties.getProperty("jdbc.driver"); +// System.out.println(">>> load driver : " + driverName); +// try { +// Class.forName(driverName); +// } catch (ClassNotFoundException e) { +// e.printStackTrace(); +// } + config.setDriverClassName(properties.getProperty("jdbc.driver")); + } else { + config.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + } + if ("com.taosdata.jdbc.rs.RestfulDriver".equalsIgnoreCase(properties.getProperty("jdbc.driver"))) + config.setJdbcUrl("jdbc:TAOS-RS://" + host + ":6041/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + else + config.setJdbcUrl("jdbc:TAOS://" + host + ":" + port + "/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + config.setUsername(user); + config.setPassword(password); + // maximum-pool-size + if (properties.containsKey("hikari.maximum-pool-size")) + config.setMaximumPoolSize(Integer.parseInt(properties.getProperty("hikari.maximum-pool-size"))); + else + config.setMaximumPoolSize(500); + // minimum-idle + if (properties.containsKey("hikari.minimum-idle")) + config.setMinimumIdle(Integer.parseInt(properties.getProperty("hikari.minimum-idle"))); + else + config.setMinimumIdle(100); + config.setMaxLifetime(0); + instance = new HikariDataSource(config); + } + } + } + return instance; + } + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/JdbcTaosdemoConfig.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java similarity index 83% rename from tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/JdbcTaosdemoConfig.java rename to tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java index 4e6f64033012ca402d4a07787ef71a0525b25acb..971c10dee2889543e95a70b244ea3cda462df3a6 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java @@ -1,4 +1,6 @@ -package com.taosdata.taosdemo.utils; +package com.taosdata.taosdemo.components; + +import com.taosdata.taosdemo.utils.TimeStampUtil; public final class JdbcTaosdemoConfig { // instance @@ -7,10 +9,14 @@ public final class JdbcTaosdemoConfig { public String user = "root"; //user public String password = "taosdata"; //password // database - public String database = "test"; //database + public String database = "jdbcdb"; //database public int keep = 3650; //keep public int days = 30; //days public int replica = 1; //replica + public int blocks = 16; + public int cache = 8; + public String precision = "ms"; + //super table public boolean doCreateTable = true; public String superTable = "weather"; //super table name @@ -20,19 +26,19 @@ public final class JdbcTaosdemoConfig { public int numOfTags; public String superTableSQL; //sub table - public String tablePrefix = "t"; - public int numOfTables = 100; - public int numOfThreadsForCreate = 1; + public String prefixOfTable = "t"; // insert task public boolean autoCreateTable; - public int numOfRowsPerTable = 100; + public long numOfTables = 10; + public long numOfRowsPerTable = 10; + public int numOfTablesPerSQL = 1; + public int numOfValuesPerSQL = 1; + public int numOfThreadsForCreate = 1; public int numOfThreadsForInsert = 1; - public int numOfTablesPerSQL = 10; - public int numOfValuesPerSQL = 10; public long startTime; - public long timeGap; - public int sleep = 0; - public int order = 0; + public long timeGap = 1; + public int frequency; + public int order; public int rate = 10; public long range = 1000l; // select task @@ -48,10 +54,14 @@ public final class JdbcTaosdemoConfig { System.out.println("-user The TDengine user name to use when connecting to the server. Default is 'root'"); System.out.println("-password The password to use when connecting to the server.Default is 'taosdata'"); // database - System.out.println("-database Destination database. Default is 'test'"); + System.out.println("-database Destination database. Default is 'jdbcdb'"); System.out.println("-keep database keep parameter. Default is 3650"); System.out.println("-days database days parameter. Default is 30"); System.out.println("-replica database replica parameter. Default 1, min: 1, max: 3"); + System.out.println("-blocks database blocks parameter. Default is 16"); + System.out.println("-cache database cache parameter. Default is 8"); + System.out.println("-precision database precision parameter. Default is ms"); + // super table System.out.println("-doCreateTable do create super table and sub table, true or false, Default true"); System.out.println("-superTable super table name. Default 'weather'"); @@ -63,7 +73,7 @@ public final class JdbcTaosdemoConfig { " Default is 'create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int). \n" + " if you use this parameter, the numOfFields and numOfTags will be invalid'"); // sub table - System.out.println("-tablePrefix The prefix of sub tables. Default is 't'"); + System.out.println("-prefixOfTable The prefix of sub tables. Default is 't'"); System.out.println("-numOfTables The number of tables. Default is 1"); System.out.println("-numOfThreadsForCreate The number of thread during create sub table. Default is 1"); // insert task @@ -74,11 +84,10 @@ public final class JdbcTaosdemoConfig { System.out.println("-numOfValuesPerSQL The number of value per SQL. Default is 1"); System.out.println("-startTime start time for insert task, The format is \"yyyy-MM-dd HH:mm:ss.SSS\"."); System.out.println("-timeGap the number of time gap. Default is 1000 ms"); - System.out.println("-sleep The number of milliseconds for sleep after each insert. default is 0"); + System.out.println("-frequency the number of records per second inserted into one table. default is 0, do not control frequency"); System.out.println("-order Insert mode--0: In order, 1: Out of order. Default is in order"); System.out.println("-rate The proportion of data out of order. effective only if order is 1. min 0, max 100, default is 10"); System.out.println("-range The range of data out of order. effective only if order is 1. default is 1000 ms"); - // query task // System.out.println("-sqlFile The select sql file"); // drop task @@ -120,6 +129,15 @@ public final class JdbcTaosdemoConfig { if ("-replica".equals(args[i]) && i < args.length - 1) { replica = Integer.parseInt(args[++i]); } + if ("-blocks".equals(args[i]) && i < args.length - 1) { + blocks = Integer.parseInt(args[++i]); + } + if ("-cache".equals(args[i]) && i < args.length - 1) { + cache = Integer.parseInt(args[++i]); + } + if ("-precision".equals(args[i]) && i < args.length - 1) { + precision = args[++i]; + } // super table if ("-doCreateTable".equals(args[i]) && i < args.length - 1) { doCreateTable = Boolean.parseBoolean(args[++i]); @@ -143,11 +161,11 @@ public final class JdbcTaosdemoConfig { superTableSQL = args[++i]; } // sub table - if ("-tablePrefix".equals(args[i]) && i < args.length - 1) { - tablePrefix = args[++i]; + if ("-prefixOfTable".equals(args[i]) && i < args.length - 1) { + prefixOfTable = args[++i]; } if ("-numOfTables".equals(args[i]) && i < args.length - 1) { - numOfTables = Integer.parseInt(args[++i]); + numOfTables = Long.parseLong(args[++i]); } if ("-autoCreateTable".equals(args[i]) && i < args.length - 1) { autoCreateTable = Boolean.parseBoolean(args[++i]); @@ -157,7 +175,7 @@ public final class JdbcTaosdemoConfig { } // insert task if ("-numOfRowsPerTable".equals(args[i]) && i < args.length - 1) { - numOfRowsPerTable = Integer.parseInt(args[++i]); + numOfRowsPerTable = Long.parseLong(args[++i]); } if ("-numOfThreadsForInsert".equals(args[i]) && i < args.length - 1) { numOfThreadsForInsert = Integer.parseInt(args[++i]); @@ -174,8 +192,8 @@ public final class JdbcTaosdemoConfig { if ("-timeGap".equals(args[i]) && i < args.length - 1) { timeGap = Long.parseLong(args[++i]); } - if ("-sleep".equals(args[i]) && i < args.length - 1) { - sleep = Integer.parseInt(args[++i]); + if ("-frequency".equals(args[i]) && i < args.length - 1) { + frequency = Integer.parseInt(args[++i]); } if ("-order".equals(args[i]) && i < args.length - 1) { order = Integer.parseInt(args[++i]); @@ -197,8 +215,4 @@ public final class JdbcTaosdemoConfig { } } - public static void main(String[] args) { - JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); - } - } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JsonConfig.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JsonConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..1c44610095f4b383f82a74dfdc11030a28afb246 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JsonConfig.java @@ -0,0 +1,39 @@ +package com.taosdata.taosdemo.components; + +import com.alibaba.fastjson.JSONObject; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +public class JsonConfig { + + public static void main(String[] args) { + + JsonConfig config = new JsonConfig(); + String str = config.read("insert.json"); + JSONObject jsonObject = JSONObject.parseObject(str); + System.out.println(jsonObject); + + } + + private String read(String fileName) { + try { + BufferedReader reader = new BufferedReader( + new InputStreamReader(JsonConfig.class.getClassLoader().getResourceAsStream(fileName)) + ); + StringBuilder sb = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + return sb.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + + return fileName; + } + + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/TaosDemoCommandLineRunner.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/TaosDemoCommandLineRunner.java deleted file mode 100644 index e58c68f7a5f60b57f064075c4004267918526b15..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/TaosDemoCommandLineRunner.java +++ /dev/null @@ -1,174 +0,0 @@ -package com.taosdata.taosdemo.components; - -import com.taosdata.taosdemo.domain.*; -import com.taosdata.taosdemo.service.DatabaseService; -import com.taosdata.taosdemo.service.SubTableService; -import com.taosdata.taosdemo.service.SuperTableService; -import com.taosdata.taosdemo.service.data.SubTableMetaGenerator; -import com.taosdata.taosdemo.service.data.SubTableValueGenerator; -import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator; -import com.taosdata.taosdemo.utils.JdbcTaosdemoConfig; -import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.CommandLineRunner; -import org.springframework.stereotype.Component; - -import java.util.*; -import java.util.concurrent.TimeUnit; - - -@Component -public class TaosDemoCommandLineRunner implements CommandLineRunner { - - private static Logger logger = Logger.getLogger(TaosDemoCommandLineRunner.class); - @Autowired - private DatabaseService databaseService; - @Autowired - private SuperTableService superTableService; - @Autowired - private SubTableService subTableService; - - private SuperTableMeta superTableMeta; - private List subTableMetaList; - private List subTableValueList; - private List> dataList; - - - @Override - public void run(String... args) throws Exception { - // 读配置参数 - JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); - boolean isHelp = Arrays.asList(args).contains("--help"); - if (isHelp) { - JdbcTaosdemoConfig.printHelp(); - System.exit(0); - } - // 准备数据 - prepareData(config); - // 创建数据库 - createDatabaseTask(config); - // 建表 - createTableTask(config); - // 插入 - insertTask(config); - // 查询: 1. 生成查询语句, 2. 执行查询 - // 删除表 - if (config.dropTable) { - superTableService.drop(config.database, config.superTable); - } - - System.exit(0); - } - - private void createDatabaseTask(JdbcTaosdemoConfig config) { - long start = System.currentTimeMillis(); - - Map databaseParam = new HashMap<>(); - databaseParam.put("database", config.database); - databaseParam.put("keep", Integer.toString(config.keep)); - databaseParam.put("days", Integer.toString(config.days)); - databaseParam.put("replica", Integer.toString(config.replica)); - //TODO: other database parameters - databaseService.dropDatabase(config.database); - databaseService.createDatabase(databaseParam); - databaseService.useDatabase(config.database); - - long end = System.currentTimeMillis(); - logger.info(">>> insert time cost : " + (end - start) + " ms."); - } - - // 建超级表，三种方式：1. 指定SQL，2. 指定field和tags的个数，3. 默认 - private void createTableTask(JdbcTaosdemoConfig config) { - long start = System.currentTimeMillis(); - if (config.doCreateTable) { - superTableService.create(superTableMeta); - // 批量建子表 - subTableService.createSubTable(subTableMetaList, config.numOfThreadsForCreate); - } - long end = System.currentTimeMillis(); - logger.info(">>> create table time cost : " + (end - start) + " ms."); - } - - private void insertTask(JdbcTaosdemoConfig config) { - long start = System.currentTimeMillis(); - - int numOfThreadsForInsert = config.numOfThreadsForInsert; - int sleep = config.sleep; - if (config.autoCreateTable) { - // 批量插入，自动建表 - dataList.stream().forEach(subTableValues -> { - subTableService.insertAutoCreateTable(subTableValues, numOfThreadsForInsert); - sleep(sleep); - }); - } else { - dataList.stream().forEach(subTableValues -> { - subTableService.insert(subTableValues, numOfThreadsForInsert); - sleep(sleep); - }); - } - long end = System.currentTimeMillis(); - logger.info(">>> insert time cost : " + (end - start) + " ms."); - } - - private void prepareData(JdbcTaosdemoConfig config) { - long start = System.currentTimeMillis(); - // 超级表的meta - superTableMeta = createSupertable(config); - // 子表的meta - subTableMetaList = SubTableMetaGenerator.generate(superTableMeta, config.numOfTables, config.tablePrefix); - // 子表的data - subTableValueList = SubTableValueGenerator.generate(subTableMetaList, config.numOfRowsPerTable, config.startTime, config.timeGap); - // 如果有乱序，给数据搞乱 - if (config.order != 0) { - SubTableValueGenerator.disrupt(subTableValueList, config.rate, config.range); - } - // 分割数据 - int numOfTables = config.numOfTables; - int numOfTablesPerSQL = config.numOfTablesPerSQL; - int numOfRowsPerTable = config.numOfRowsPerTable; - int numOfValuesPerSQL = config.numOfValuesPerSQL; - dataList = SubTableValueGenerator.split(subTableValueList, numOfTables, numOfTablesPerSQL, numOfRowsPerTable, numOfValuesPerSQL); - long end = System.currentTimeMillis(); - logger.info(">>> prepare data time cost : " + (end - start) + " ms."); - } - - private SuperTableMeta createSupertable(JdbcTaosdemoConfig config) { - SuperTableMeta tableMeta; - // create super table - logger.info(">>> create super table <<<"); - if (config.superTableSQL != null) { - // use a sql to create super table - tableMeta = SuperTableMetaGenerator.generate(config.superTableSQL); - } else if (config.numOfFields == 0) { - // default sql = "create table test.weather (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; - SuperTableMeta superTableMeta = new SuperTableMeta(); - superTableMeta.setDatabase(config.database); - superTableMeta.setName(config.superTable); - List fields = new ArrayList<>(); - fields.add(new FieldMeta("ts", "timestamp")); - fields.add(new FieldMeta("temperature", "float")); - fields.add(new FieldMeta("humidity", "int")); - superTableMeta.setFields(fields); - List tags = new ArrayList<>(); - tags.add(new TagMeta("location", "nchar(64)")); - tags.add(new TagMeta("groupId", "int")); - superTableMeta.setTags(tags); - return superTableMeta; - } else { - // create super table with specified field size and tag size - tableMeta = SuperTableMetaGenerator.generate(config.database, config.superTable, config.numOfFields, config.prefixOfFields, config.numOfTags, config.prefixOfTags); - } - return tableMeta; - } - - private static void sleep(int sleep) { - if (sleep <= 0) - return; - try { - TimeUnit.MILLISECONDS.sleep(sleep); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/DatabaseController.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/DatabaseController.java deleted file mode 100644 index 1cf1463f0ab4f2a8c67258f512e407d54c1d869e..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/DatabaseController.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.taosdata.taosdemo.controller; - -import com.taosdata.taosdemo.service.DatabaseService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -import java.util.Map; - -@RestController -@RequestMapping -public class DatabaseController { - - @Autowired - private DatabaseService databaseService; - - /** - * create database - ***/ - @PostMapping - public int create(@RequestBody Map map) { - return databaseService.createDatabase(map); - } - - - /** - * drop database - **/ - @DeleteMapping("/{dbname}") - public int delete(@PathVariable("dbname") String dbname) { - return databaseService.dropDatabase(dbname); - } - - /** - * use database - **/ - @GetMapping("/{dbname}") - public int use(@PathVariable("dbname") String dbname) { - return databaseService.useDatabase(dbname); - } -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/InsertController.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/InsertController.java deleted file mode 100644 index 788f68a30af5ac7dd4c8c27f4cfe5f1a6f2b440b..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/InsertController.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.taosdata.taosdemo.controller; - -import org.springframework.web.bind.annotation.RestController; - -@RestController -public class InsertController { - - //TODO：多线程写一张表, thread = 10, table = 1 - //TODO：一个批次写多张表, insert into t1 using weather values() t2 using weather values() - //TODO：插入的频率， - //TODO：指定一张表内的records数量 - //TODO：是否乱序， - //TODO：乱序的比例，乱序的范围 - //TODO：先建表，自动建表 - //TODO：一个批次写多张表 - -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SubTableController.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SubTableController.java deleted file mode 100644 index 797c3708d3357c19bf5f64046bb794b66786e080..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SubTableController.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.taosdata.taosdemo.controller; - -import com.taosdata.taosdemo.domain.TableValue; -import com.taosdata.taosdemo.service.SuperTableService; -import com.taosdata.taosdemo.service.TableService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; - -@RestController -public class SubTableController { - - @Autowired - private TableService tableService; - @Autowired - private SuperTableService superTableService; - - //TODO: 使用supertable创建一个子表 - - //TODO：使用supertable创建多个子表 - - //TODO：使用supertable多线程创建子表 - - //TODO：使用supertable多线程创建子表，指定子表的name_prefix，子表的数量，使用线程的个数 - - /** - * 创建表，超级表或者普通表 - **/ - - - /** - * 创建超级表的子表 - **/ - @PostMapping("/{database}/{superTable}") - public int createTable(@PathVariable("database") String database, - @PathVariable("superTable") String superTable, - @RequestBody TableValue tableMetadta) { - tableMetadta.setDatabase(database); - return 0; - } - - -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SuperTableController.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SuperTableController.java deleted file mode 100644 index cf53c1440fa710c3bea510973ca2efecb0705828..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/SuperTableController.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.taosdata.taosdemo.controller; - -import com.taosdata.taosdemo.domain.SuperTableMeta; -import com.taosdata.taosdemo.service.SuperTableService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; - -public class SuperTableController { - @Autowired - private SuperTableService superTableService; - - - @PostMapping("/{database}") - public int createTable(@PathVariable("database") String database, @RequestBody SuperTableMeta tableMetadta) { - tableMetadta.setDatabase(database); - return superTableService.create(tableMetadta); - } - - //TODO: 删除超级表 - - //TODO：查询超级表 - - //TODO：统计查询表 -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/TableController.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/TableController.java deleted file mode 100644 index dbdd978e746eb89d8927617ae4026cdcf544559b..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/controller/TableController.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.taosdata.taosdemo.controller; - -public class TableController { - - //TODO：创建普通表，create table(ts timestamp, temperature float) - - //TODO：创建普通表，指定表的列数，包括第一列timestamp - - //TODO：创建普通表，指定表每列的name和type - -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..56e38d2bfce8ba5801d0f8dc48093d198852fbca --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapper.java @@ -0,0 +1,23 @@ +package com.taosdata.taosdemo.dao; + +import java.util.Map; + +public interface DatabaseMapper { + + // create database if not exists XXX + void createDatabase(String dbname); + + // drop database if exists XXX + void dropDatabase(String dbname); + + // create database if not exists XXX keep XX days XX replica XX + void createDatabaseWithParameters(Map map); + + // use XXX + void useDatabase(String dbname); + + //TODO: alter database + + //TODO: show database + +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..69bae160f6285bb9d1a94357cc2279ebfca22eca --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java @@ -0,0 +1,47 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; +import java.util.Map; + +public class DatabaseMapperImpl implements DatabaseMapper { + private static final Logger logger = Logger.getLogger(DatabaseMapperImpl.class); + + private final JdbcTemplate jdbcTemplate; + + public DatabaseMapperImpl(DataSource dataSource) { + this.jdbcTemplate = new JdbcTemplate(dataSource); + } + + + @Override + public void createDatabase(String dbname) { + String sql = "create database if not exists " + dbname; + jdbcTemplate.execute(sql); + logger.info("SQL >>> " + sql); + } + + @Override + public void dropDatabase(String dbname) { + String sql = "drop database if exists " + dbname; + jdbcTemplate.update(sql); + logger.info("SQL >>> " + sql); + } + + @Override + public void createDatabaseWithParameters(Map map) { + String sql = SqlSpeller.createDatabase(map); + jdbcTemplate.execute(sql); + logger.info("SQL >>> " + sql); + } + + @Override + public void useDatabase(String dbname) { + String sql = "use " + dbname; + jdbcTemplate.execute(sql); + logger.info("SQL >>> " + sql); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java similarity index 67% rename from tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.java rename to tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java index d23473ba313899eee04f07f5c06177a2d55f6295..e0ddd220c19066afd1cc332f007a82e2fdab2b07 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapper.java @@ -1,8 +1,7 @@ -package com.taosdata.taosdemo.mapper; +package com.taosdata.taosdemo.dao; import com.taosdata.taosdemo.domain.SubTableMeta; import com.taosdata.taosdemo.domain.SubTableValue; -import org.apache.ibatis.annotations.Param; import org.springframework.stereotype.Repository; import java.util.List; @@ -11,7 +10,7 @@ import java.util.List; public interface SubTableMapper { // 创建：子表 - int createUsingSuperTable(SubTableMeta subTableMeta); + void createUsingSuperTable(SubTableMeta subTableMeta); // 插入：一张子表多个values int insertOneTableMultiValues(SubTableValue subTableValue); @@ -20,10 +19,10 @@ public interface SubTableMapper { int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue); // 插入：多张表多个values - int insertMultiTableMultiValues(@Param("tables") List tables); + int insertMultiTableMultiValues(List tables); // 插入：多张表多个values，自动建表 - int insertMultiTableMultiValuesUsingSuperTable(@Param("tables") List tables); + int insertMultiTableMultiValuesUsingSuperTable(List tables); // diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..e3a6691430f8b5232660ed6edeab01318db30ef1 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java @@ -0,0 +1,81 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.SubTableMeta; +import com.taosdata.taosdemo.domain.SubTableValue; +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; +import java.util.List; + +public class SubTableMapperImpl implements SubTableMapper { + + private static final Logger logger = Logger.getLogger(SubTableMapperImpl.class); + private final JdbcTemplate jdbcTemplate; + + public SubTableMapperImpl(DataSource dataSource) { + this.jdbcTemplate = new JdbcTemplate(dataSource); + } + + @Override + public void createUsingSuperTable(SubTableMeta subTableMeta) { + String sql = SqlSpeller.createTableUsingSuperTable(subTableMeta); + logger.info("SQL >>> " + sql); + jdbcTemplate.execute(sql); + } + + @Override + public int insertOneTableMultiValues(SubTableValue subTableValue) { + String sql = SqlSpeller.insertOneTableMultiValues(subTableValue); + logger.info("SQL >>> " + sql); + + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } + + @Override + public int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue) { + String sql = SqlSpeller.insertOneTableMultiValuesUsingSuperTable(subTableValue); + logger.info("SQL >>> " + sql); + + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } + + @Override + public int insertMultiTableMultiValues(List tables) { + String sql = SqlSpeller.insertMultiSubTableMultiValues(tables); + logger.info("SQL >>> " + sql); + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } + + @Override + public int insertMultiTableMultiValuesUsingSuperTable(List tables) { + String sql = SqlSpeller.insertMultiTableMultiValuesUsingSuperTable(tables); + logger.info("SQL >>> " + sql); + int affectRows = 0; + try { + affectRows = jdbcTemplate.update(sql); + } catch (Exception e) { + e.printStackTrace(); + } + return affectRows; + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java similarity index 70% rename from tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.java rename to tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java index c8610fac901efa907971d700bd9956cac264236f..9f8cec9e8fa5af8741d9efbdce72f240aa7a09aa 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapper.java @@ -1,20 +1,16 @@ -package com.taosdata.taosdemo.mapper; +package com.taosdata.taosdemo.dao; import com.taosdata.taosdemo.domain.SuperTableMeta; -import org.apache.ibatis.annotations.Param; import org.springframework.stereotype.Repository; @Repository public interface SuperTableMapper { - // 创建超级表，使用自己定义的SQL语句 - int createSuperTableUsingSQL(@Param("createSuperTableSQL") String sql); - // 创建超级表 create table if not exists xxx.xxx (f1 type1, f2 type2, ... ) tags( t1 type1, t2 type2 ...) - int createSuperTable(SuperTableMeta tableMetadata); + void createSuperTable(SuperTableMeta tableMetadata); // 删除超级表 drop table if exists xxx; - int dropSuperTable(@Param("database") String database, @Param("name") String name); + void dropSuperTable(String database, String name); // diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..a293de5100d9af33bc88ad5f02ae632333a5b5b8 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java @@ -0,0 +1,31 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; + +import javax.sql.DataSource; + +public class SuperTableMapperImpl implements SuperTableMapper { + private static final Logger logger = Logger.getLogger(SuperTableMapperImpl.class); + private JdbcTemplate jdbcTemplate; + + public SuperTableMapperImpl(DataSource dataSource) { + this.jdbcTemplate = new JdbcTemplate(dataSource); + } + + @Override + public void createSuperTable(SuperTableMeta tableMetadata) { + String sql = SqlSpeller.createSuperTable(tableMetadata); + logger.info("SQL >>> " + sql); + jdbcTemplate.execute(sql); + } + + @Override + public void dropSuperTable(String database, String name) { + String sql = "drop table if exists " + database + "." + name; + logger.info("SQL >>> " + sql); + jdbcTemplate.execute(sql); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java similarity index 65% rename from tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.java rename to tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java index f00f6c96941d47258d91a34ffeb1de99c4c8435e..32d1875e4d1a82f7dfb658d68ed0e63a5cbfa040 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapper.java @@ -1,8 +1,7 @@ -package com.taosdata.taosdemo.mapper; +package com.taosdata.taosdemo.dao; import com.taosdata.taosdemo.domain.TableMeta; import com.taosdata.taosdemo.domain.TableValue; -import org.apache.ibatis.annotations.Param; import org.springframework.stereotype.Repository; import java.util.List; @@ -11,7 +10,7 @@ import java.util.List; public interface TableMapper { // 创建：普通表 - int create(TableMeta tableMeta); + void create(TableMeta tableMeta); // 插入：一张表多个value int insertOneTableMultiValues(TableValue values); @@ -20,9 +19,9 @@ public interface TableMapper { int insertOneTableMultiValuesWithColumns(TableValue values); // 插入：多个表多个value - int insertMultiTableMultiValues(@Param("tables") List tables); + int insertMultiTableMultiValues(List tables); // 插入：多个表多个value, 指定的列 - int insertMultiTableMultiValuesWithColumns(@Param("tables") List tables); + int insertMultiTableMultiValuesWithColumns(List tables); } \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..77415619f02a1a0650448e9a8856fcb8d33e4921 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java @@ -0,0 +1,43 @@ +package com.taosdata.taosdemo.dao; + +import com.taosdata.taosdemo.dao.TableMapper; +import com.taosdata.taosdemo.domain.TableMeta; +import com.taosdata.taosdemo.domain.TableValue; +import com.taosdata.taosdemo.utils.SqlSpeller; +import org.springframework.jdbc.core.JdbcTemplate; + +import java.util.List; + +public class TableMapperImpl implements TableMapper { + private JdbcTemplate template; + + @Override + public void create(TableMeta tableMeta) { + String sql = SqlSpeller.createTable(tableMeta); + template.execute(sql); + } + + @Override + public int insertOneTableMultiValues(TableValue values) { + String sql = SqlSpeller.insertOneTableMultiValues(values); + return template.update(sql); + } + + @Override + public int insertOneTableMultiValuesWithColumns(TableValue values) { + String sql = SqlSpeller.insertOneTableMultiValuesWithColumns(values); + return template.update(sql); + } + + @Override + public int insertMultiTableMultiValues(List tables) { + String sql = SqlSpeller.insertMultiTableMultiValues(tables); + return template.update(sql); + } + + @Override + public int insertMultiTableMultiValuesWithColumns(List tables) { + String sql = SqlSpeller.insertMultiTableMultiValuesWithColumns(tables); + return template.update(sql); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java index a9f216f6792820931af4570450318ddc53e3c8b7..a444fa78dcdeb8f1bb76974a29051c98348a055b 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/domain/RowValue.java @@ -8,7 +8,6 @@ import java.util.List; public class RowValue { private List fields; - public RowValue(List fields) { this.fields = fields; } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.java deleted file mode 100644 index e535ed1f985d37688c8461e1f0bef3a803f541d4..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.taosdata.taosdemo.mapper; - -import org.apache.ibatis.annotations.Param; -import org.springframework.stereotype.Repository; - -import java.util.Map; - -@Repository -public interface DatabaseMapper { - - // create database if not exists XXX - int createDatabase(@Param("database") String dbname); - - // drop database if exists XXX - int dropDatabase(@Param("database") String dbname); - - // create database if not exists XXX keep XX days XX replica XX - int createDatabaseWithParameters(Map map); - - // use XXX - int useDatabase(@Param("database") String dbname); - - //TODO: alter database - - //TODO: show database - -} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.xml b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.xml deleted file mode 100644 index 1a1de348428ae63ad16f9988f535f339f61d8c25..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/DatabaseMapper.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - create database if not exists ${database} - - - - DROP database if exists ${database} - - - - CREATE database if not exists ${database} - - KEEP ${keep} - - - DAYS ${days} - - - REPLICA ${replica} - - - cache ${cache} - - - blocks ${blocks} - - - minrows ${minrows} - - - maxrows ${maxrows} - - - - - use ${database} - - - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.xml b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.xml deleted file mode 100644 index 2fb94e99b77623ab8731c03f398058dbdc459d07..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SubTableMapper.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - - - - - CREATE table IF NOT EXISTS ${database}.${name} USING ${supertable} TAGS - - #{tag.value} - - - - - - INSERT INTO ${database}.${name} - VALUES - - - #{field.value} - - - - - - - INSERT INTO ${database}.${name} USING ${supertable} TAGS - - #{tag.value} - - VALUES - - - #{field.value} - - - - - - - - - - - INSERT INTO - - ${table.database}.${table.name} - VALUES - - - #{field.value} - - - - - - - - INSERT INTO - - ${table.database}.${table.name} USING ${table.supertable} TAGS - - #{tag.value} - - VALUES - - - #{field.value} - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.xml b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.xml deleted file mode 100644 index 8b83d57a4bcbf7a2a6b3f69911b3accc61064480..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/SuperTableMapper.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - ${createSuperTableSQL} - - - - - create table if not exists ${database}.${name} - - ${field.name} ${field.type} - - tags - - ${tag.name} ${tag.type} - - - - - - drop table if exists ${database}.${name} - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.xml b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.xml deleted file mode 100644 index e2e7cbb30def21f88ec7615b7a6dd4a769cb643a..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/mapper/TableMapper.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - create table if not exists ${database}.${name} - - ${field.name} ${field.type} - - - - - - insert into ${database}.${name} values - - - ${field.value} - - - - - - - insert into ${database}.${name} - - ${column.name} - - values - - - ${field.value} - - - - - - - insert into - - ${table.database}.${table.name} values - - - ${field.value} - - - - - - - - insert into - - ${table.database}.${table.name} - - ${column.name} - - values - - - ${field.value} - - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java index e9aa2727a06ade0ff3d7661f84977c64f64ea9af..3c8e9624066bf629a74ebcaa3959a2e15338c363 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/DatabaseService.java @@ -1,38 +1,42 @@ package com.taosdata.taosdemo.service; -import com.taosdata.taosdemo.mapper.DatabaseMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; +import com.taosdata.taosdemo.dao.DatabaseMapper; +import com.taosdata.taosdemo.dao.DatabaseMapperImpl; +import javax.sql.DataSource; import java.util.Map; -@Service public class DatabaseService { - @Autowired - private DatabaseMapper databaseMapper; + private final DatabaseMapper databaseMapper; + + public DatabaseService(DataSource dataSource) { + this.databaseMapper = new DatabaseMapperImpl(dataSource); + } // 建库，指定 name - public int createDatabase(String database) { - return databaseMapper.createDatabase(database); + public void createDatabase(String database) { + databaseMapper.createDatabase(database); } // 建库，指定参数 keep,days,replica等 - public int createDatabase(Map map) { + public void createDatabase(Map map) { if (map.isEmpty()) - return 0; - if (map.containsKey("database") && map.size() == 1) - return databaseMapper.createDatabase(map.get("database")); - return databaseMapper.createDatabaseWithParameters(map); + return; + if (map.containsKey("database") && map.size() == 1) { + createDatabase(map.get("database")); + return; + } + databaseMapper.createDatabaseWithParameters(map); } // drop database - public int dropDatabase(String dbname) { - return databaseMapper.dropDatabase(dbname); + public void dropDatabase(String dbname) { + databaseMapper.dropDatabase(dbname); } // use database - public int useDatabase(String dbname) { - return databaseMapper.useDatabase(dbname); + public void useDatabase(String dbname) { + databaseMapper.useDatabase(dbname); } } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java index 07c315b65a9e07222fe85066c3f1f3b1edb107fe..cea98a1c5d350ed22ed5d26c72fedb212dcb7f26 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java @@ -1,68 +1,63 @@ package com.taosdata.taosdemo.service; +import com.taosdata.taosdemo.components.JdbcTaosdemoConfig; +import com.taosdata.taosdemo.dao.SubTableMapper; +import com.taosdata.taosdemo.dao.SubTableMapperImpl; import com.taosdata.taosdemo.domain.SubTableMeta; import com.taosdata.taosdemo.domain.SubTableValue; -import com.taosdata.taosdemo.mapper.SubTableMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; +import com.taosdata.taosdemo.domain.SuperTableMeta; +import com.taosdata.taosdemo.service.data.SubTableMetaGenerator; +import com.taosdata.taosdemo.service.data.SubTableValueGenerator; +import org.apache.log4j.Logger; +import javax.sql.DataSource; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; -@Service public class SubTableService extends AbstractService { - @Autowired private SubTableMapper mapper; + private static final Logger logger = Logger.getLogger(SubTableService.class); - /** - * 1. 选择database，找到所有supertable - * 2. 选择supertable，可以拿到表结构，包括field和tag - * 3. 指定子表的前缀和个数 - * 4. 指定创建子表的线程数 - */ - //TODO：指定database、supertable、子表前缀、子表个数、线程数 + public SubTableService(DataSource datasource) { + this.mapper = new SubTableMapperImpl(datasource); + } - // 多线程创建表，指定线程个数 - public int createSubTable(List subTables, int threadSize) { - ExecutorService executor = Executors.newFixedThreadPool(threadSize); - List> futureList = new ArrayList<>(); - for (SubTableMeta subTableMeta : subTables) { - Future future = executor.submit(() -> createSubTable(subTableMeta)); - futureList.add(future); + public void createSubTable(SuperTableMeta superTableMeta, long numOfTables, String prefixOfTable, int numOfThreadsForCreate) { + ExecutorService executor = Executors.newFixedThreadPool(numOfThreadsForCreate); + for (long i = 0; i < numOfTables; i++) { + long tableIndex = i; + executor.execute(() -> createSubTable(superTableMeta, prefixOfTable + (tableIndex + 1))); } executor.shutdown(); - return getAffectRows(futureList); + try { + executor.awaitTermination(Long.MAX_VALUE,TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } } - - // 创建一张子表，可以指定database，supertable，tablename，tag值 - public int createSubTable(SubTableMeta subTableMeta) { - return mapper.createUsingSuperTable(subTableMeta); + public void createSubTable(SuperTableMeta superTableMeta, String tableName) { + // 构造数据 + SubTableMeta meta = SubTableMetaGenerator.generate(superTableMeta, tableName); + createSubTable(meta); } - // 单线程创建多张子表，每张子表分别可以指定自己的database，supertable，tablename，tag值 - public int createSubTable(List subTables) { - return createSubTable(subTables, 1); + // 创建一张子表，可以指定database，supertable，tablename，tag值 + public void createSubTable(SubTableMeta subTableMeta) { + mapper.createUsingSuperTable(subTableMeta); } /*************************************************************************************************************************/ // 插入：多线程，多表 - public int insert(List subTableValues, int threadSize) { + public int insert(List subTableValues, int threadSize, int frequency) { ExecutorService executor = Executors.newFixedThreadPool(threadSize); Future future = executor.submit(() -> insert(subTableValues)); executor.shutdown(); - return getAffectRows(future); - } - - // 插入：多线程，多表, 自动建表 - public int insertAutoCreateTable(List subTableValues, int threadSize) { - ExecutorService executor = Executors.newFixedThreadPool(threadSize); - Future future = executor.submit(() -> insertAutoCreateTable(subTableValues)); - executor.shutdown(); + //TODO：frequency return getAffectRows(future); } @@ -73,7 +68,7 @@ public class SubTableService extends AbstractService { // 插入: 多表，insert into xxx values(),()... xxx values(),()... public int insert(List subTableValues) { - return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues); + return mapper.insertMultiTableMultiValues(subTableValues); } // 插入：单表，自动建表, insert into xxx using xxx tags(...) values(),()... @@ -86,33 +81,128 @@ public class SubTableService extends AbstractService { return mapper.insertMultiTableMultiValuesUsingSuperTable(subTableValues); } + public int insertMultiThreads(SuperTableMeta superTableMeta, int threadSize, long tableSize, long startTime, long gap, JdbcTaosdemoConfig config) { + List taskList = new ArrayList<>(); + List threads = IntStream.range(0, threadSize) + .mapToObj(i -> { + long startInd = i * gap; + long endInd = (i + 1) * gap < tableSize ? (i + 1) * gap : tableSize; + FutureTask task = new FutureTask<>( + new InsertTask(superTableMeta, + startInd, endInd, + startTime, config.timeGap, + config.numOfRowsPerTable, config.numOfTablesPerSQL, config.numOfValuesPerSQL, + config.order, config.rate, config.range, + config.prefixOfTable, config.autoCreateTable) + ); + taskList.add(task); + return new Thread(task, "InsertThread-" + i); + }).collect(Collectors.toList()); + + threads.stream().forEach(Thread::start); + for (Thread thread : threads) { + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + int affectedRows = 0; + for (FutureTask task : taskList) { + try { + affectedRows += task.get(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + + return affectedRows; + } -// ExecutorService executors = Executors.newFixedThreadPool(threadSize); -// int count = 0; -// -// // -// List subTableValues = new ArrayList<>(); -// for (int tableIndex = 1; tableIndex <= numOfTablesPerSQL; tableIndex++) { -// // each table -// SubTableValue subTableValue = new SubTableValue(); -// subTableValue.setDatabase(); -// subTableValue.setName(); -// subTableValue.setSupertable(); -// -// List values = new ArrayList<>(); -// for (int valueCnt = 0; valueCnt < numOfValuesPerSQL; valueCnt++) { -// List fields = new ArrayList<>(); -// for (int fieldInd = 0; fieldInd <; fieldInd++) { -// FieldValue field = new FieldValue<>("", ""); -// fields.add(field); -// } -// RowValue row = new RowValue(); -// row.setFields(fields); -// values.add(row); -// } -// subTableValue.setValues(values); -// subTableValues.add(subTableValue); -// } + private class InsertTask implements Callable { + + private final long startTableInd; // included + private final long endTableInd; // excluded + private final long startTime; + private final long timeGap; + private final long numOfRowsPerTable; + private long numOfTablesPerSQL; + private long numOfValuesPerSQL; + private final SuperTableMeta superTableMeta; + private final int order; + private final int rate; + private final long range; + private final String prefixOfTable; + private final boolean autoCreateTable; + + public InsertTask(SuperTableMeta superTableMeta, long startTableInd, long endTableInd, + long startTime, long timeGap, + long numOfRowsPerTable, long numOfTablesPerSQL, long numOfValuesPerSQL, + int order, int rate, long range, + String prefixOfTable, boolean autoCreateTable) { + this.superTableMeta = superTableMeta; + this.startTableInd = startTableInd; + this.endTableInd = endTableInd; + this.startTime = startTime; + this.timeGap = timeGap; + this.numOfRowsPerTable = numOfRowsPerTable; + this.numOfTablesPerSQL = numOfTablesPerSQL; + this.numOfValuesPerSQL = numOfValuesPerSQL; + this.order = order; + this.rate = rate; + this.range = range; + this.prefixOfTable = prefixOfTable; + this.autoCreateTable = autoCreateTable; + } + + + @Override + public Integer call() { + + long numOfTables = endTableInd - startTableInd; + if (numOfRowsPerTable < numOfValuesPerSQL) + numOfValuesPerSQL = (int) numOfRowsPerTable; + if (numOfTables < numOfTablesPerSQL) + numOfTablesPerSQL = (int) numOfTables; + + int affectRows = 0; + // row + for (long rowCnt = 0; rowCnt < numOfRowsPerTable; ) { + long rowSize = numOfValuesPerSQL; + if (rowCnt + rowSize > numOfRowsPerTable) { + rowSize = numOfRowsPerTable - rowCnt; + } + //table + for (long tableCnt = startTableInd; tableCnt < endTableInd; ) { + long tableSize = numOfTablesPerSQL; + if (tableCnt + tableSize > endTableInd) { + tableSize = endTableInd - tableCnt; + } + long startTime = this.startTime + rowCnt * timeGap; +// System.out.println(Thread.currentThread().getName() + " >>> " + "rowCnt: " + rowCnt + ", rowSize: " + rowSize + ", " + "tableCnt: " + tableCnt + ",tableSize: " + tableSize + ", " + "startTime: " + startTime + ",timeGap: " + timeGap + ""); + /***********************************************/ + // 生成数据 + List data = SubTableValueGenerator.generate(superTableMeta, prefixOfTable, tableCnt, tableSize, rowSize, startTime, timeGap); + // 乱序 + if (order != 0) + SubTableValueGenerator.disrupt(data, rate, range); + // insert + if (autoCreateTable) + affectRows += insertAutoCreateTable(data); + else + affectRows += insert(data); + /***********************************************/ + tableCnt += tableSize; + } + rowCnt += rowSize; + } + + return affectRows; + } + } } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java index 7f6836c999ba0eb2e6eaab94e9e026f3856d8faf..b91348e2d008bc1ac32faffc5912a8509adf42bd 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SuperTableService.java @@ -1,19 +1,22 @@ package com.taosdata.taosdemo.service; +import com.taosdata.taosdemo.dao.SuperTableMapper; +import com.taosdata.taosdemo.dao.SuperTableMapperImpl; import com.taosdata.taosdemo.domain.SuperTableMeta; -import com.taosdata.taosdemo.mapper.SuperTableMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -@Service +import javax.sql.DataSource; + public class SuperTableService { - @Autowired private SuperTableMapper superTableMapper; + public SuperTableService(DataSource dataSource) { + this.superTableMapper = new SuperTableMapperImpl(dataSource); + } + // 创建超级表，指定每个field的名称和类型，每个tag的名称和类型 - public int create(SuperTableMeta superTableMeta) { - return superTableMapper.createSuperTable(superTableMeta); + public void create(SuperTableMeta superTableMeta) { + superTableMapper.createSuperTable(superTableMeta); } public void drop(String database, String name) { diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java index bada6de708e44e04237a7e6a3b734eb46974e21f..b4ad2d17e58a3f7c04665707f0cd3e7327d7c16c 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java @@ -1,41 +1,25 @@ package com.taosdata.taosdemo.service; +import com.taosdata.taosdemo.dao.TableMapper; import com.taosdata.taosdemo.domain.TableMeta; -import com.taosdata.taosdemo.mapper.TableMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; @Service public class TableService extends AbstractService { - @Autowired private TableMapper tableMapper; //创建一张表 - public int create(TableMeta tableMeta) { - return tableMapper.create(tableMeta); + public void create(TableMeta tableMeta) { + tableMapper.create(tableMeta); } //创建多张表 - public int create(List tables) { - return create(tables, 1); - } - - //多线程创建多张表 - public int create(List tables, int threadSize) { - ExecutorService executors = Executors.newFixedThreadPool(threadSize); - List> futures = new ArrayList<>(); - for (TableMeta table : tables) { - Future future = executors.submit(() -> create(table)); - futures.add(future); - } - return getAffectRows(futures); + public void create(List tables) { + tables.stream().forEach(this::create); } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java index d15ad0d8bdd066a9dea50251094657d20fdc4fd7..88e3c0d26a5a7558c1c07f0fc38ae21710438dbe 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableMetaGenerator.java @@ -27,4 +27,16 @@ public class SubTableMetaGenerator { return subTableMetaList; } + public static SubTableMeta generate(SuperTableMeta superTableMeta, String tableName) { + SubTableMeta subTableMeta = new SubTableMeta(); + // create table xxx.xxx using xxx tags(...) + subTableMeta.setDatabase(superTableMeta.getDatabase()); + subTableMeta.setName(tableName); + subTableMeta.setSupertable(superTableMeta.getName()); + subTableMeta.setFields(superTableMeta.getFields()); + List tagValues = TagValueGenerator.generate(superTableMeta.getTags()); + subTableMeta.setTags(tagValues); + return subTableMeta; + } + } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java index a36f718f83e20e2659322f0f23d6375ebfa3af0d..8c318dbd3abf2ddfec8b51e83f32246421c49d51 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/data/SubTableValueGenerator.java @@ -1,8 +1,6 @@ package com.taosdata.taosdemo.service.data; -import com.taosdata.taosdemo.domain.RowValue; -import com.taosdata.taosdemo.domain.SubTableMeta; -import com.taosdata.taosdemo.domain.SubTableValue; +import com.taosdata.taosdemo.domain.*; import com.taosdata.taosdemo.utils.TimeStampUtil; import org.springframework.beans.BeanUtils; @@ -11,22 +9,26 @@ import java.util.List; public class SubTableValueGenerator { - public static List generate(List subTableMetaList, int numOfRowsPerTable, long start, long timeGap) { - List subTableValueList = new ArrayList<>(); - - subTableMetaList.stream().forEach((subTableMeta) -> { - // insert into xxx.xxx using xxxx tags(...) values(),()... + public static List generate(SuperTableMeta superTableMeta, String prefixOfTables, long tableIndex, long tableSize, long valueSize, long startTime, long timeGap) { + List subTableValues = new ArrayList<>(); + for (int i = 1; i <= tableSize; i++) { SubTableValue subTableValue = new SubTableValue(); - subTableValue.setDatabase(subTableMeta.getDatabase()); - subTableValue.setName(subTableMeta.getName()); - subTableValue.setSupertable(subTableMeta.getSupertable()); - subTableValue.setTags(subTableMeta.getTags()); - TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(start, timeGap, numOfRowsPerTable); - List values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, subTableMeta.getFields()); + subTableValue.setDatabase(superTableMeta.getDatabase()); + subTableValue.setName(prefixOfTables + (tableIndex + i)); + subTableValue.setSupertable(superTableMeta.getName()); + TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(startTime, timeGap, valueSize); + List tags = TagValueGenerator.generate(superTableMeta.getTags()); + subTableValue.setTags(tags); + List values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, superTableMeta.getFields()); subTableValue.setValues(values); - subTableValueList.add(subTableValue); - }); - return subTableValueList; + + subTableValues.add(subTableValue); + } + return subTableValues; + } + + public static List generate(List subTableMetaList, int numOfRowsPerTable, long start, long timeGap) { + return generate(subTableMetaList, 0, subTableMetaList.size(), numOfRowsPerTable, start, timeGap); } public static void disrupt(List subTableValueList, int rate, long range) { @@ -38,12 +40,10 @@ public class SubTableValueGenerator { public static List> split(List subTableValueList, int numOfTables, int numOfTablesPerSQL, int numOfRowsPerTable, int numOfValuesPerSQL) { List> dataList = new ArrayList<>(); - if (numOfRowsPerTable < numOfValuesPerSQL) numOfValuesPerSQL = numOfRowsPerTable; if (numOfTables < numOfTablesPerSQL) numOfTablesPerSQL = numOfTables; - //table for (int tableCnt = 0; tableCnt < numOfTables; ) { int tableSize = numOfTablesPerSQL; @@ -81,4 +81,20 @@ public class SubTableValueGenerator { split(null, 99, 10, 99, 10); } + public static List generate(List subTableMetaList, int tableCnt, int tableSize, int rowSize, long startTime, long timeGap) { + List subTableValueList = new ArrayList<>(); + for (int i = 0; i < tableSize; i++) { + SubTableMeta subTableMeta = subTableMetaList.get(tableCnt + i); + SubTableValue subTableValue = new SubTableValue(); + subTableValue.setDatabase(subTableMeta.getDatabase()); + subTableValue.setName(subTableMeta.getName()); + subTableValue.setSupertable(subTableMeta.getSupertable()); + subTableValue.setTags(subTableMeta.getTags()); + TimeStampUtil.TimeTuple tuple = TimeStampUtil.range(startTime, timeGap, rowSize); + List values = FieldValueGenerator.generate(tuple.start, tuple.end, tuple.timeGap, subTableMeta.getFields()); + subTableValue.setValues(values); + subTableValueList.add(subTableValue); + } + return subTableValueList; + } } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java new file mode 100644 index 0000000000000000000000000000000000000000..a60f0641d3a4441195c3a60639fbe3a197115dc3 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/SqlSpeller.java @@ -0,0 +1,194 @@ +package com.taosdata.taosdemo.utils; + +import com.taosdata.taosdemo.domain.*; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class SqlSpeller { + + // create database if not exists xxx keep xx days xx replica xx cache xx... + public static String createDatabase(Map map) { + StringBuilder sb = new StringBuilder(); + sb.append("create database if not exists ").append(map.get("database")).append(" "); + if (map.containsKey("keep")) + sb.append("keep ").append(map.get("keep")).append(" "); + if (map.containsKey("days")) + sb.append("days ").append(map.get("days")).append(" "); + if (map.containsKey("replica")) + sb.append("replica ").append(map.get("replica")).append(" "); + if (map.containsKey("cache")) + sb.append("cache ").append(map.get("cache")).append(" "); + if (map.containsKey("blocks")) + sb.append("blocks ").append(map.get("blocks")).append(" "); + if (map.containsKey("minrows")) + sb.append("minrows ").append(map.get("minrows")).append(" "); + if (map.containsKey("maxrows")) + sb.append("maxrows ").append(map.get("maxrows")).append(" "); + if (map.containsKey("precision")) + sb.append("precision ").append(map.get("precision")).append(" "); + if (map.containsKey("comp")) + sb.append("comp ").append(map.get("comp")).append(" "); + if (map.containsKey("walLevel")) + sb.append("walLevel ").append(map.get("walLevel")).append(" "); + if (map.containsKey("quorum")) + sb.append("quorum ").append(map.get("quorum")).append(" "); + if (map.containsKey("fsync")) + sb.append("fsync ").append(map.get("fsync")).append(" "); + if (map.containsKey("update")) + sb.append("update ").append(map.get("update")).append(" "); + return sb.toString(); + } + + // create table if not exists xx.xx using xx.xx tags(x,x,x) + public static String createTableUsingSuperTable(SubTableMeta subTableMeta) { + StringBuilder sb = new StringBuilder(); + sb.append("create table if not exists ").append(subTableMeta.getDatabase()).append(".").append(subTableMeta.getName()).append(" "); + sb.append("using ").append(subTableMeta.getDatabase()).append(".").append(subTableMeta.getSupertable()).append(" "); +// String tagStr = subTableMeta.getTags().stream().filter(Objects::nonNull) +// .map(tagValue -> tagValue.getName() + " '" + tagValue.getValue() + "' ") +// .collect(Collectors.joining(",", "(", ")")); + sb.append("tags ").append(tagValues(subTableMeta.getTags())); + return sb.toString(); + } + + // insert into xx.xxx values(x,x,x),(x,x,x)... + public static String insertOneTableMultiValues(SubTableValue subTableValue) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getName() + " "); + sb.append("values").append(rowValues(subTableValue.getValues())); + return sb.toString(); + } + + //f1, f2, f3 + private static String fieldValues(List fields) { + return IntStream.range(0, fields.size()).mapToObj(i -> { + if (i == 0) { + return "" + fields.get(i).getValue() + ""; + } else { + return "'" + fields.get(i).getValue() + "'"; + } + }).collect(Collectors.joining(",", "(", ")")); + +// return fields.stream() +// .filter(Objects::nonNull) +// .map(fieldValue -> "'" + fieldValue.getValue() + "'") +// .collect(Collectors.joining(",", "(", ")")); + } + + //(f1, f2, f3),(f1, f2, f3) + private static String rowValues(List rowValues) { + return rowValues.stream().filter(Objects::nonNull) + .map(rowValue -> fieldValues(rowValue.getFields())) + .collect(Collectors.joining(",", "", "")); + } + + // insert into xx.xxx using xx.xx tags(x,x,x) values(x,x,x),(x,x,x)... + public static String insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getName()).append(" "); + sb.append("using ").append(subTableValue.getDatabase()).append(".").append(subTableValue.getSupertable()).append(" "); + sb.append("tags ").append(tagValues(subTableValue.getTags()) + " "); + sb.append("values ").append(rowValues(subTableValue.getValues())); + return sb.toString(); + } + + // (t1,t2,t3...) + private static String tagValues(List tags) { + return tags.stream().filter(Objects::nonNull) + .map(tagValue -> "'" + tagValue.getValue() + "'") + .collect(Collectors.joining(",", "(", ")")); + } + + // insert into xx.xx values(),(),()... xx.xx values(),()... + public static String insertMultiSubTableMultiValues(List tables) { + return "insert into " + tables.stream().filter(Objects::nonNull) + .map(table -> table.getDatabase() + "." + table.getName() + " values " + rowValues(table.getValues())) + .collect(Collectors.joining(" ", "", "")); + } + + // insert into xx.xx using xx.xx tags(xx,xx) values(),()... + public static String insertMultiTableMultiValuesUsingSuperTable(List tables) { + return "insert into " + tables.stream().filter(Objects::nonNull) + .map(table -> { + StringBuilder sb = new StringBuilder(); + sb.append(table.getDatabase()).append(".").append(table.getName()); + sb.append(" using ").append(table.getDatabase()).append(".").append(table.getSupertable()); + sb.append(" tags ").append(tagValues(table.getTags())); + sb.append(" values ").append(rowValues(table.getValues())); + return sb.toString(); + }).collect(Collectors.joining(" ")); + } + + // create table if not exists xx.xx (f1 xx,f2 xx...) tags(t1 xx, t2 xx...) + public static String createSuperTable(SuperTableMeta tableMetadata) { + StringBuilder sb = new StringBuilder(); + sb.append("create table if not exists ").append(tableMetadata.getDatabase()).append(".").append(tableMetadata.getName()); + String fields = tableMetadata.getFields().stream() + .filter(Objects::nonNull).map(field -> field.getName() + " " + field.getType() + " ") + .collect(Collectors.joining(",", "(", ")")); + sb.append(fields); + sb.append(" tags "); + String tags = tableMetadata.getTags().stream().filter(Objects::nonNull) + .map(tag -> tag.getName() + " " + tag.getType() + " ") + .collect(Collectors.joining(",", "(", ")")); + sb.append(tags); + return sb.toString(); + } + + + public static String createTable(TableMeta tableMeta) { + StringBuilder sb = new StringBuilder(); + sb.append("create table if not exists ").append(tableMeta.getDatabase()).append(".").append(tableMeta.getName()).append(" "); + String fields = tableMeta.getFields().stream() + .filter(Objects::nonNull).map(field -> field.getName() + " " + field.getType() + " ") + .collect(Collectors.joining(",", "(", ")")); + sb.append(fields); + return sb.toString(); + } + + // insert into xx.xx values() + public static String insertOneTableMultiValues(TableValue table) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table.getDatabase()).append(".").append(table.getName() + " "); + sb.append("values").append(rowValues(table.getValues())); + return sb.toString(); + + } + + // insert into xx.xx (f1, f2, f3...) values(xx,xx,xx),(xx,xx,xx)... + public static String insertOneTableMultiValuesWithColumns(TableValue table) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table.getDatabase()).append(".").append(table.getName()).append(" "); + sb.append(columnNames(table.getColumns())); + sb.append(" values ").append(rowValues(table.getValues())); + return sb.toString(); + } + + // (f1, f2, f3...) + private static String columnNames(List fields) { + return fields.stream() + .filter(Objects::nonNull) + .map(column -> column.getName() + " ") + .collect(Collectors.joining(",", "(", ")")); + } + + public static String insertMultiTableMultiValuesWithColumns(List tables) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(tables.stream().filter(Objects::nonNull) + .map(table -> table.getDatabase() + "." + table.getName() + " " + columnNames(table.getColumns()) + " values " + rowValues(table.getValues())) + .collect(Collectors.joining(" "))); + return sb.toString(); + } + + public static String insertMultiTableMultiValues(List tables) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(tables.stream().filter(Objects::nonNull).map(table -> + table.getDatabase() + "." + table.getName() + " values " + rowValues(table.getValues()) + ).collect(Collectors.joining(" "))); + return sb.toString(); + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/application.properties b/tests/examples/JDBC/taosdemo/src/main/resources/application.properties index 1e7a7de89fa495b8898a58805bd9e256db1d69ad..6fd38f1762e6879422cf26fe2dc617caeb5e297c 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/taosdemo/src/main/resources/application.properties @@ -1,14 +1,5 @@ -#spring.datasource.url=jdbc:mysql://master:3306/?useSSL=false&useUnicode=true&characterEncoding=UTF-8 -#spring.datasource.driver-class-name=com.mysql.jdbc.Driver -#spring.datasource.username=root -#spring.datasource.password=123456 - -spring.datasource.url=jdbc:TAOS://master:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8 -spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -spring.datasource.username=root -spring.datasource.password=taosdata - -spring.datasource.hikari.maximum-pool-size=10 -spring.datasource.hikari.minimum-idle=10 -spring.datasource.hikari.max-lifetime=600000 -logging.level.com.taosdata.taosdemo.mapper=debug \ No newline at end of file +jdbc.driver=com.taosdata.jdbc.rs.RestfulDriver +#jdbc.driver=com.taosdata.jdbc.TSDBDriver +hikari.maximum-pool-size=1 +hikari.minimum-idle=1 +hikari.max-lifetime=0 \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..a7bd87e6d3bbf9f6ec1b0a68d31c4da6c620c994 --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json @@ -0,0 +1,119 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 2, + "databases": [ + { + "dbinfo": { + "name": "db04", + "drop": "no", + "replica": 1, + "days": 2, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "quorum": 1, + "fsync": 3000, + "update": 0 + }, + "super_tables": [ + { + "name": "stb04", + "child_table_exists": "no", + "childtable_count": 10, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 100, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 3, + "max_sql_len": 1024, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + { + "type": "TINYINT" + }, + { + "type": "SMALLINT" + }, + { + "type": "INT" + }, + { + "type": "BIGINT" + }, + { + "type": "BOOL" + }, + { + "type": "FLOAT" + }, + { + "type": "DOUBLE" + }, + { + "type": "TIMESTAMP" + }, + { + "type": "BINARY", + "len": 16 + }, + { + "type": "NCHAR", + "len": 4 + } + ], + "tags": [ + { + "type": "TINYINT" + }, + { + "type": "SMALLINT" + }, + { + "type": "INT" + }, + { + "type": "BIGINT" + }, + { + "type": "BOOL" + }, + { + "type": "FLOAT" + }, + { + "type": "DOUBLE" + }, + { + "type": "BINARY", + "len": 16 + }, + { + "type": "NCHAR", + "len": 4 + } + ] + } + ] + } + ] +} diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar b/tests/examples/JDBC/taosdemo/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar new file mode 100644 index 0000000000000000000000000000000000000000..58508b92e5afec7e400691d213940536e8abe5f6 Binary files /dev/null and b/tests/examples/JDBC/taosdemo/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar differ diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties b/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties index 1299357be3d2e99ca6b79227f14ca7a587718914..b2a9586ea78e2d55a0091097097988b038267405 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties +++ b/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties @@ -1,5 +1,5 @@ ### 设置### -log4j.rootLogger=debug,stdout,DebugLog,ErrorLog +log4j.rootLogger=error,stdout ### 输出信息到控制抬 ### log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/query.json b/tests/examples/JDBC/taosdemo/src/main/resources/query.json new file mode 100644 index 0000000000000000000000000000000000000000..53d0b319212196257aa3e84be1221bd6e2bd0d8d --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/resources/query.json @@ -0,0 +1,17 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db01", + "super_table_query": + {"rate":1, "concurrent":1, + "sqls": [{"sql": "select count(*) from stb01", "result": "./query_res0.txt"}] + }, + "sub_table_query": + {"stblname": "stb01", "rate":1, "threads":1, + "sqls": [{"sql": "select count(*) from xxxx", "result": "./query_res1.txt"}] + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/templates/index.html b/tests/examples/JDBC/taosdemo/src/main/resources/templates/index.html index 69f8851c9b83eaf2768331ea713c212fe500992d..953a058032612c7eac8eacc68f738d5fef03ec39 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/templates/index.html +++ b/tests/examples/JDBC/taosdemo/src/main/resources/templates/index.html @@ -5,6 +5,6 @@ Index -

Hello~~~

+

Developing~~~

\ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/TaosdemoApplicationTests.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/TaosdemoApplicationTests.java deleted file mode 100644 index e8725091873c2dad5cb9e4a8c2735bf7418f4ef5..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/TaosdemoApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.taosdata.taosdemo; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class TaosdemoApplicationTests { - - @Test - void contextLoads() { - } - -} diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/DatabaseMapperTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/DatabaseMapperTest.java deleted file mode 100644 index 8364e16ed06b8d9a4ae74cd370281c19149989d2..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/DatabaseMapperTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.taosdata.taosdemo.mapper; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.util.HashMap; -import java.util.Map; - -@RunWith(SpringRunner.class) -@SpringBootTest -public class DatabaseMapperTest { - @Autowired - private DatabaseMapper databaseMapper; - - @Test - public void createDatabase() { - databaseMapper.createDatabase("db_test"); - } - - @Test - public void dropDatabase() { - databaseMapper.dropDatabase("db_test"); - } - - @Test - public void creatDatabaseWithParameters() { - Map map = new HashMap<>(); - map.put("dbname", "weather"); - map.put("keep", "3650"); - map.put("days", "30"); - map.put("replica", "1"); - databaseMapper.createDatabaseWithParameters(map); - } - - @Test - public void useDatabase() { - databaseMapper.useDatabase("test"); - } -} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/SubTableMapperTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/SubTableMapperTest.java deleted file mode 100644 index 90faa20496c5ecfe64c929910df4028099f974ad..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/SubTableMapperTest.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.taosdata.taosdemo.mapper; - -import com.taosdata.taosdemo.domain.*; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.util.ArrayList; -import java.util.List; - -@RunWith(SpringRunner.class) -@SpringBootTest -public class SubTableMapperTest { - @Autowired - private SubTableMapper subTableMapper; - private List tables; - - @Test - public void createUsingSuperTable() { - SubTableMeta subTableMeta = new SubTableMeta(); - subTableMeta.setDatabase("test"); - subTableMeta.setSupertable("weather"); - subTableMeta.setName("t1"); - List tags = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - tags.add(new TagValue("tag" + (i + 1), "nchar(64)")); - } - subTableMeta.setTags(tags); - subTableMapper.createUsingSuperTable(subTableMeta); - } - - @Test - public void insertOneTableMultiValues() { - subTableMapper.insertOneTableMultiValues(tables.get(0)); - } - - @Test - public void insertOneTableMultiValuesUsingSuperTable() { - subTableMapper.insertOneTableMultiValuesUsingSuperTable(tables.get(0)); - } - - - @Test - public void insertMultiTableMultiValues() { - subTableMapper.insertMultiTableMultiValues(tables); - } - - @Test - public void insertMultiTableMultiValuesUsingSuperTable() { - subTableMapper.insertMultiTableMultiValuesUsingSuperTable(tables); - } - - - @Before - public void before() { - tables = new ArrayList<>(); - for (int ind = 0; ind < 3; ind++) { - - SubTableValue table = new SubTableValue(); - table.setDatabase("test"); - // supertable - table.setSupertable("weather"); - table.setName("t" + (ind + 1)); - // tags - List tags = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - tags.add(new TagValue("tag" + (i + 1), "beijing")); - } - table.setTags(tags); - // values - List values = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - List fields = new ArrayList<>(); - for (int j = 0; j < 4; j++) { - fields.add(new FieldValue("f" + (j + 1), (j + 1) * 10)); - } - values.add(new RowValue(fields)); - } - table.setValues(values); - - tables.add(table); - } - } - -} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/SuperTableMapperTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/SuperTableMapperTest.java deleted file mode 100644 index 6c97874cfc00b128f4a3e09d9d0f698274baeff0..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/SuperTableMapperTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.taosdata.taosdemo.mapper; - -import com.taosdata.taosdemo.domain.FieldMeta; -import com.taosdata.taosdemo.domain.SuperTableMeta; -import com.taosdata.taosdemo.domain.TagMeta; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.util.ArrayList; -import java.util.List; - -@RunWith(SpringRunner.class) -@SpringBootTest -public class SuperTableMapperTest { - @Autowired - private SuperTableMapper superTableMapper; - - @Test - public void testCreateSuperTableUsingSQL() { - String sql = "create table test.weather (ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; - superTableMapper.createSuperTableUsingSQL(sql); - } - - @Test - public void createSuperTable() { - SuperTableMeta superTableMeta = new SuperTableMeta(); - superTableMeta.setDatabase("test"); - superTableMeta.setName("weather"); - List fields = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - fields.add(new FieldMeta("f" + (i + 1), "int")); - } - superTableMeta.setFields(fields); - List tags = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - tags.add(new TagMeta("t" + (i + 1), "nchar(64)")); - } - superTableMeta.setTags(tags); - - superTableMapper.createSuperTable(superTableMeta); - } - - @Test - public void dropSuperTable() { - superTableMapper.dropSuperTable("test", "weather"); - } -} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/TableMapperTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/TableMapperTest.java deleted file mode 100644 index 3a051b311268ba076f97edc1b36032eac0b52709..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/mapper/TableMapperTest.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.taosdata.taosdemo.mapper; - -import com.taosdata.taosdemo.domain.*; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -@SpringBootTest -@RunWith(SpringRunner.class) -public class TableMapperTest { - @Autowired - private TableMapper tableMapper; - private static Random random = new Random(System.currentTimeMillis()); - - @Test - public void create() { - TableMeta table = new TableMeta(); - table.setDatabase("test"); - table.setName("t1"); - List fields = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - FieldMeta field = new FieldMeta(); - field.setName("f" + (i + 1)); - field.setType("nchar(64)"); - fields.add(field); - } - table.setFields(fields); - tableMapper.create(table); - } - - @Test - public void insertOneTableMultiValues() { - TableValue table = new TableValue(); - table.setDatabase("test"); - table.setName("t1"); - List values = new ArrayList<>(); - for (int j = 0; j < 5; j++) { - List fields = new ArrayList<>(); - for (int k = 0; k < 2; k++) { - FieldValue field = new FieldValue<>(); - field.setValue((k + 1) * 100); - fields.add(field); - } - values.add(new RowValue(fields)); - } - table.setValues(values); - - tableMapper.insertOneTableMultiValues(table); - } - - @Test - public void insertOneTableMultiValuesWithCoulmns() { - TableValue tableValue = new TableValue(); - tableValue.setDatabase("test"); - tableValue.setName("weather"); - // columns - List columns = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - FieldMeta field = new FieldMeta(); - field.setName("f" + (i + 1)); - columns.add(field); - } - tableValue.setColumns(columns); - // values - List values = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - List fields = new ArrayList<>(); - for (int j = 0; j < 3; j++) { - FieldValue field = new FieldValue(); - field.setValue(j); - fields.add(field); - } - values.add(new RowValue(fields)); - } - tableValue.setValues(values); - tableMapper.insertOneTableMultiValuesWithColumns(tableValue); - } - - @Test - public void insertMultiTableMultiValues() { - List tables = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - TableValue table = new TableValue(); - table.setDatabase("test"); - table.setName("t" + (i + 1)); - List values = new ArrayList<>(); - for (int j = 0; j < 5; j++) { - List fields = new ArrayList<>(); - for (int k = 0; k < 2; k++) { - FieldValue field = new FieldValue<>(); - field.setValue((k + 1) * 10); - fields.add(field); - } - values.add(new RowValue(fields)); - } - table.setValues(values); - - tables.add(table); - } - tableMapper.insertMultiTableMultiValues(tables); - } - - @Test - public void insertMultiTableMultiValuesWithCoulumns() { - List tables = new ArrayList<>(); - for (int i = 0; i < 3; i++) { - TableValue table = new TableValue(); - table.setDatabase("test"); - table.setName("t" + (i + 1)); - // columns - List columns = new ArrayList<>(); - for (int j = 0; j < 3; j++) { - FieldMeta field = new FieldMeta(); - field.setName("f" + (j + 1)); - columns.add(field); - } - table.setColumns(columns); - // values - List values = new ArrayList<>(); - for (int j = 0; j < 5; j++) { - List fields = new ArrayList<>(); - for (int k = 0; k < columns.size(); k++) { - FieldValue field = new FieldValue<>(); - field.setValue((k + 1) * 10); - fields.add(field); - } - values.add(new RowValue(fields)); - } - table.setValues(values); - - tables.add(table); - } - tableMapper.insertMultiTableMultiValuesWithColumns(tables); - } - -} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java index 2c1cdf6e00695aa6fa26395c17458e66626e62ea..621ba7df5da4bb29747a5a4af1f91d51f6a1d7d4 100644 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/DatabaseServiceTest.java @@ -1,15 +1,9 @@ package com.taosdata.taosdemo.service; import org.junit.Test; -import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; -@RunWith(SpringRunner.class) -@SpringBootTest public class DatabaseServiceTest { - @Autowired private DatabaseService service; @Test diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SubTableServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SubTableServiceTest.java index 4e54de3f1385561107aedbe48d89362961783759..f7e5cd45057472602ad6e7a43a8d8bdb31a02b40 100644 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SubTableServiceTest.java +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SubTableServiceTest.java @@ -4,18 +4,12 @@ import com.taosdata.taosdemo.domain.SubTableMeta; import com.taosdata.taosdemo.domain.TagValue; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; import java.util.ArrayList; import java.util.List; -@RunWith(SpringRunner.class) -@SpringBootTest public class SubTableServiceTest { - @Autowired + private SubTableService service; private List subTables; @@ -38,13 +32,11 @@ public class SubTableServiceTest { @Test public void testCreateSubTable() { - int count = service.createSubTable(subTables); - System.out.println("count >>> " + count); + } @Test public void testCreateSubTableList() { - int count = service.createSubTable(subTables, 10); - System.out.println("count >>> " + count); + } } \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java index b9291fceaf89d3fdeef4c638b4ace5c995b7e369..33e52af1eaa779e7ed402a63633bf0dbb9fbadd7 100644 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/SuperTableServiceTest.java @@ -4,19 +4,12 @@ import com.taosdata.taosdemo.domain.FieldMeta; import com.taosdata.taosdemo.domain.SuperTableMeta; import com.taosdata.taosdemo.domain.TagMeta; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; import java.util.ArrayList; import java.util.List; -@RunWith(SpringRunner.class) -@SpringBootTest public class SuperTableServiceTest { - @Autowired private SuperTableService service; @Test diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java index fdbd5546294ce43bc147ddad314d1b9498f1d5ad..1f52198d68823326dd81d8c419fc02d89e15ef2d 100644 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/TableServiceTest.java @@ -3,18 +3,11 @@ package com.taosdata.taosdemo.service; import com.taosdata.taosdemo.domain.TableMeta; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; import java.util.ArrayList; import java.util.List; -@RunWith(SpringRunner.class) -@SpringBootTest public class TableServiceTest { - @Autowired private TableService tableService; private List tables; @@ -32,12 +25,7 @@ public class TableServiceTest { @Test public void testCreate() { - int count = tableService.create(tables); - System.out.println(count); + tableService.create(tables); } - @Test - public void testCreateMultiThreads() { - System.out.println(tableService.create(tables, 10)); - } } \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/SqlSpellerTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/SqlSpellerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..daabd51ca75a6c9f4bfeead0b747c4de69f40a7a --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/SqlSpellerTest.java @@ -0,0 +1,254 @@ +package com.taosdata.taosdemo.utils; + +import com.taosdata.taosdemo.domain.*; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class SqlSpellerTest { + + @Test + public void createDatabase() { + HashMap map = new HashMap<>(); + map.put("database", "jdbcdb"); + map.put("keep", "3650"); + map.put("days", "30"); + map.put("replica", "1"); + map.put("minRows", "100"); + map.put("maxRows", "1000"); + map.put("cache", "16"); + map.put("blocks", "8"); + map.put("precision", "ms"); + map.put("comp", "2"); + map.put("walLevel", "1"); + map.put("quorum", "1"); + map.put("fsync", "3000"); + map.put("update", "0"); + String sql = SqlSpeller.createDatabase(map); + System.out.println(sql); + } + + @Test + public void createTableUsingSuperTable() { + SubTableMeta subTableMeta = new SubTableMeta(); + subTableMeta.setDatabase("test"); + subTableMeta.setSupertable("weather"); + subTableMeta.setName("t1"); + List tags = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + tags.add(new TagValue("tag" + (i + 1), "nchar(64)")); + } + subTableMeta.setTags(tags); + String sql = SqlSpeller.createTableUsingSuperTable(subTableMeta); + System.out.println(sql); + } + + @Test + public void insertOneTableMultiValues() { + String sql = SqlSpeller.insertOneTableMultiValues(tables.get(0)); + System.out.println(sql); + } + + @Test + public void insertOneTableMultiValuesUsingSuperTable() { + String sql = SqlSpeller.insertOneTableMultiValuesUsingSuperTable(tables.get(0)); + System.out.println(sql); + } + + @Test + public void insertMultiTableMultiValues() { + String sql = SqlSpeller.insertMultiSubTableMultiValues(tables); + System.out.println(sql); + } + + @Test + public void insertMultiTableMultiValuesUsingSuperTable() { + String sql = SqlSpeller.insertMultiTableMultiValuesUsingSuperTable(tables); + System.out.println(sql); + } + + private List tables; + + @Before + public void before() { + tables = new ArrayList<>(); + for (int ind = 0; ind < 3; ind++) { + SubTableValue table = new SubTableValue(); + table.setDatabase("test"); + // supertable + table.setSupertable("weather"); + table.setName("t" + (ind + 1)); + // tags + List tags = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + tags.add(new TagValue("tag" + (i + 1), "beijing")); + } + table.setTags(tags); + // values + List values = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + List fields = new ArrayList<>(); + for (int j = 0; j < 4; j++) { + fields.add(new FieldValue("f" + (j + 1), (j + 1) * 10)); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + + tables.add(table); + } + } + + @Test + public void createSuperTable() { + SuperTableMeta superTableMeta = new SuperTableMeta(); + superTableMeta.setDatabase("test"); + superTableMeta.setName("weather"); + List fields = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + fields.add(new FieldMeta("f" + (i + 1), "int")); + } + superTableMeta.setFields(fields); + List tags = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + tags.add(new TagMeta("t" + (i + 1), "nchar(64)")); + } + superTableMeta.setTags(tags); + + String sql = SqlSpeller.createSuperTable(superTableMeta); + System.out.println(sql); + } + + @Test + public void createTable() { + TableMeta table = new TableMeta(); + table.setDatabase("test"); + table.setName("t1"); + List fields = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + FieldMeta field = new FieldMeta(); + field.setName("f" + (i + 1)); + field.setType("nchar(64)"); + fields.add(field); + } + table.setFields(fields); + String sql = SqlSpeller.createTable(table); + System.out.println(sql); + } + + + @Test + public void testInsertOneTableMultiValues() { + TableValue table = new TableValue(); + table.setDatabase("test"); + table.setName("t1"); + List values = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + List fields = new ArrayList<>(); + for (int k = 0; k < 2; k++) { + FieldValue field = new FieldValue<>(); + field.setValue((k + 1) * 100); + fields.add(field); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + + String sql = SqlSpeller.insertOneTableMultiValues(table); + System.out.println(sql); + } + + @Test + public void insertOneTableMultiValuesWithColumns() { + TableValue tableValue = new TableValue(); + tableValue.setDatabase("test"); + tableValue.setName("weather"); + // columns + List columns = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + FieldMeta field = new FieldMeta(); + field.setName("f" + (i + 1)); + columns.add(field); + } + tableValue.setColumns(columns); + // values + List values = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + List fields = new ArrayList<>(); + for (int j = 0; j < 3; j++) { + FieldValue field = new FieldValue(); + field.setValue(j); + fields.add(field); + } + values.add(new RowValue(fields)); + } + tableValue.setValues(values); + + String sql = SqlSpeller.insertOneTableMultiValuesWithColumns(tableValue); + System.out.println(sql); + } + + @Test + public void insertMultiTableMultiValuesWithColumns() { + List tables = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + TableValue table = new TableValue(); + table.setDatabase("test"); + table.setName("t" + (i + 1)); + // columns + List columns = new ArrayList<>(); + for (int j = 0; j < 3; j++) { + FieldMeta field = new FieldMeta(); + field.setName("f" + (j + 1)); + columns.add(field); + } + table.setColumns(columns); + // values + List values = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + List fields = new ArrayList<>(); + for (int k = 0; k < columns.size(); k++) { + FieldValue field = new FieldValue<>(); + field.setValue((k + 1) * 10); + fields.add(field); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + tables.add(table); + } + + String sql = SqlSpeller.insertMultiTableMultiValuesWithColumns(tables); + System.out.println(sql); + } + + @Test + public void testInsertMultiTableMultiValues() { + List tables = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + TableValue table = new TableValue(); + table.setDatabase("test"); + table.setName("t" + (i + 1)); + List values = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + List fields = new ArrayList<>(); + for (int k = 0; k < 2; k++) { + FieldValue field = new FieldValue<>(); + field.setValue((k + 1) * 10); + fields.add(field); + } + values.add(new RowValue(fields)); + } + table.setValues(values); + + tables.add(table); + } + + String sql = SqlSpeller.insertMultiTableMultiValues(tables); + System.out.println(sql); + } + +} \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/TimeStampUtilTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/TimeStampUtilTest.java index 628594c4b18008fd9e278bb6d23d0d5f87489b56..a4845677c55897c71791eba40104ea23de644f5c 100644 --- a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/TimeStampUtilTest.java +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/utils/TimeStampUtilTest.java @@ -17,6 +17,8 @@ public class TimeStampUtilTest { @Test public void longToDatetime() { + System.out.println(TimeStampUtil.longToDatetime(1293334499006l)); + String datetime = TimeStampUtil.longToDatetime(1510000000000L); assertEquals("2017-11-07 04:26:40.000", datetime); long timestamp = TimeStampUtil.datetimeToLong(datetime); diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index b96daa54649b115961de97245c6cd109d82f0768..8498094181857095fa52123b364119801dc93d56 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -74,8 +74,14 @@ function runQueryPerfTest { CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` + AVGDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` + MAXDELAY=`echo ${delay:0:${#delay}-3}` + delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'` + MINDELAY=`echo ${delay:0:${#delay}-2}` - python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND | tee -a $PERFORMANCE_TEST_REPORT + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT [ -f taosdemoperf.txt ] && rm taosdemoperf.txt } diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 9a8c359e4e2a963e79d3afdb830728549178af5a..7bdab8bc6717905e0d81309b0c948d2f3108b55c 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -388,7 +388,9 @@ class ConcurrentInquiry: print( "Failure thread%d, sql: %s \nexception: %s" % (threadID, str(sql),str(e))) - #exit(-1) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) loop -= 1 if loop == 0: break @@ -415,7 +417,9 @@ class ConcurrentInquiry: print( "Failure thread%d, sql: %s \nexception: %s" % (threadID, str(sql),str(e))) - #exit(-1) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) loop -= 1 if loop == 0: break diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index e2ce4b26fa2fa8cc9c601218c1bf19d838b57c46..c6b857b097671b8faf650f4659c2b2990ef67d97 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2224,22 +2224,25 @@ class ClientManager: if svcMgr: # gConfig.auto_start_service: svcMgr.stopTaosServices() svcMgr = None - # Print exec status, etc., AFTER showing messages from the server - self.conclude() - # print("TC failed (2) = {}".format(self.tc.isFailed())) - # Linux return code: ref https://shapeshed.com/unix-exit-codes/ - ret = 1 if self.tc.isFailed() else 0 - self.tc.cleanup() + # Release global variables gConfig = None gSvcMgr = None logger = None + + thPool = None + dbManager.cleanUp() # destructor wouldn't run in time + dbManager = None + # Print exec status, etc., AFTER showing messages from the server + self.conclude() + # print("TC failed (2) = {}".format(self.tc.isFailed())) + # Linux return code: ref https://shapeshed.com/unix-exit-codes/ + ret = 1 if self.tc.isFailed() else 0 + self.tc.cleanup() # Release variables here self.tc = None - thPool = None - dbManager = None gc.collect() # force garbage collection # h = hpy() diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 855e18be55fd5ae0d88865644a2ebfc836b7ccf3..dc072d7abce68debd69cb162a4d784fcc5b68c4e 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -394,6 +394,7 @@ class DbManager(): cType == 'native') else DbConn.createRest(dbTarget) try: self._dbConn.open() # may throw taos.error.ProgrammingError: disconnected + Logging.debug("DbManager opened DB connection...") except taos.error.ProgrammingError as err: # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err)) if (err.msg == 'client disconnected'): # cannot open DB connection @@ -412,6 +413,10 @@ class DbManager(): # Moved to Database() # self._stateMachine = StateMechine(self._dbConn) + def __del__(self): + ''' Release the underlying DB connection upon deletion of DbManager ''' + self.cleanUp() + def getDbConn(self): return self._dbConn @@ -438,5 +443,8 @@ class DbManager(): return "table_{}".format(tblNum) def cleanUp(self): - self._dbConn.close() + if self._dbConn: + self._dbConn.close() + self._dbConn = None + Logging.debug("DbManager closed DB connection...") diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index a374ed943b41b52c2ccbee0825bca3080ed43ea9..6ea5691ce223eb1c14214d4b11c47cf85e29c795 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -2,6 +2,7 @@ import threading import random import logging import os +import sys import taos @@ -53,7 +54,7 @@ class Logging: # global misc.logger _logger = logging.getLogger('CrashGen') # real logger _logger.addFilter(LoggingFilter()) - ch = logging.StreamHandler() + ch = logging.StreamHandler(sys.stdout) # Ref: https://stackoverflow.com/questions/14058453/making-python-loggers-output-all-messages-to-stdout-in-addition-to-log-file _logger.addHandler(ch) # Logging adapter, to be used as a logger diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py index fd12284b9d7782ac7df89c37fcb653ca3bebe82b..de2d9b0780f3654424d8e077a4dda9ac5ac0ed50 100644 --- a/tests/pytest/crash_gen_bootstrap.py +++ b/tests/pytest/crash_gen_bootstrap.py @@ -19,5 +19,5 @@ if __name__ == "__main__": mExec.init() exitCode = mExec.run() - print("Exiting with code: {}".format(exitCode)) + print("\nCrash_Gen is now exiting with status code: {}".format(exitCode)) sys.exit(exitCode) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index f197a95818c54f4e142c47c1f3aa120adb9673b9..cb233f85b8132910b316fd130983769e99452bf9 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -178,11 +178,15 @@ python3 ./test.py -f query/floatCompare.py #stream python3 ./test.py -f stream/metric_1.py +python3 ./test.py -f stream/metric_n.py python3 ./test.py -f stream/new.py python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py #python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py +python3 ./test.py -f stream/sys.py +python3 ./test.py -f stream/table_1.py +python3 ./test.py -f stream/table_n.py #alter table python3 ./test.py -f alter/alter_table_crash.py @@ -217,11 +221,14 @@ python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py +python3 ./test.py -f functions/all_null_value.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py python3 client/twoClients.py -python3 test.py -f query/queryInterval.py -python3 test.py -f query/queryFillTest.py +python3 ./test.py -f query/queryInterval.py +python3 ./test.py -f query/queryFillTest.py +python3 ./test.py -f query/last_row_cache.py +python3 ./test.py -f query/last_cache.py # tools python3 test.py -f tools/taosdemoTest.py diff --git a/tests/pytest/functions/all_null_value.py b/tests/pytest/functions/all_null_value.py new file mode 100644 index 0000000000000000000000000000000000000000..5354b48f80dae25f7cd65fb00f90e4eeef2908cc --- /dev/null +++ b/tests/pytest/functions/all_null_value.py @@ -0,0 +1,90 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 int, c2 int)") + for i in range(self.rowNum): + tdSql.execute("insert into st values(%d, null, null)" % (self.ts + i)) + + tdSql.query("select avg(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select max(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select min(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select bottom(c1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select top(c1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select diff(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select first(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select last(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select last_row(c1) from st") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select count(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select leastsquares(c1, 1, 1) from st") + tdSql.checkRows(0) + + tdSql.query("select c1 + c2 from st") + tdSql.checkRows(10) + + tdSql.query("select spread(c1) from st") + tdSql.checkRows(0) + + # tdSql.query("select stddev(c1) from st") + # tdSql.checkRows(0) + + tdSql.query("select sum(c1) from st") + tdSql.checkRows(0) + + tdSql.query("select twa(c1) from st") + tdSql.checkRows(0) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index 4074166f927acbbcbb5899a692ff87f3c187f92f..3cc389221845b38d92ca130e086d50fddc344d6d 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -85,6 +85,22 @@ class TDTestCase: tdSql.checkData(0, 1, 0.1) tdSql.checkData(1, 1, 1.1) + #TD-2457 bottom + interval + order by + tdSql.error('select top(col2,1) from test interval(1y) order by col2;') + + #TD-2563 top + super_table + interval + tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") + tdSql.execute("create table t1 using meters tags (1)") + sql = 'insert into t1 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i , i % 47) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t1 values ' + tdSql.execute(sql) + tdSql.query('select bottom(c,1) from meters interval(10a)') + tdSql.checkData(0,1,0) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/functions/function_first.py b/tests/pytest/functions/function_first.py index 2b78bd33d8bdb0bcd6775215b1cf0b1e398f2928..5b2aacb779a64377baa0417d5ba0ba5ef0a27290 100644 --- a/tests/pytest/functions/function_first.py +++ b/tests/pytest/functions/function_first.py @@ -110,6 +110,10 @@ class TDTestCase: tdSql.query("select first(col9) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, '涛思数据1') + + # TD-2607 first,last + where none exist condition + interval + tdSql.query("select first(*),last(*) from test1 where ts < 23 interval(1s)") + tdSql.checkRows(0) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py index 3bd5031276adfd76574a1a43e311a1d6ec81ab16..d1f8f75892ce2e61388c107eb7cad1ed00582d2f 100644 --- a/tests/pytest/functions/function_max.py +++ b/tests/pytest/functions/function_max.py @@ -69,6 +69,15 @@ class TDTestCase: tdSql.query("select max(col6) from test1") tdSql.checkData(0, 0, np.max(floatData)) + + # test case: https://jira.taosdata.com:18080/browse/TD-2583 + tdSql.execute("create database test days 2") + tdSql.execute("create table car(ts timestamp, speed int)") + tdSql.execute("insert into car values(now, -1)") + tdSql.execute("insert into car values(now-10d, null)") + + tdSql.query("select max(speed) from car") + tdSql.checkData(0, 0, -1) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py index bc180bc22414a4a3aa3d233438b10a04fd08bc2b..c779744ced63d83fd5b2116fb61dfa347ec4dd5a 100644 --- a/tests/pytest/functions/function_min.py +++ b/tests/pytest/functions/function_min.py @@ -69,6 +69,15 @@ class TDTestCase: tdSql.query("select min(col6) from test1") tdSql.checkData(0, 0, np.min(floatData)) + + # test case: https://jira.taosdata.com:18080/browse/TD-2583 + tdSql.execute("create database test days 2") + tdSql.execute("create table car(ts timestamp, speed int)") + tdSql.execute("insert into car values(now, 1)") + tdSql.execute("insert into car values(now-10d, null)") + + tdSql.query("select min(speed) from car") + tdSql.checkData(0, 0, 1) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py index e63d65f2e6a429015e2b4d7dcbe5e8c9884eea5e..e793008ce681a04111fdee29e91520fed944d05a 100644 --- a/tests/pytest/functions/function_percentile.py +++ b/tests/pytest/functions/function_percentile.py @@ -142,6 +142,14 @@ class TDTestCase: tdSql.error("select percentile(voltage, 20) from meters") tdSql.query("select apercentile(voltage, 20) from meters") print("apercentile result: %s" % tdSql.getData(0, 0)) + + # Test case for: https://jira.taosdata.com:18080/browse/TD-2609 + tdSql.execute("create table st(ts timestamp, k int)") + tdSql.execute("insert into st values(now, -100)") + tdSql.query("select apercentile(k, 20) from st") + tdSql.checkData(0, 0, -100.00) + + def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py index e24ff1cc5339c79e1a87653e9b528ec34f3c2ea9..a98f31eea0f48114be00d3ca93bb93a1622d8470 100644 --- a/tests/pytest/functions/function_top.py +++ b/tests/pytest/functions/function_top.py @@ -89,6 +89,20 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 8.1) tdSql.checkData(1, 1, 9.1) + + #TD-2563 top + super_table + interval + tdSql.execute("create table meters(ts timestamp, c int) tags (d int)") + tdSql.execute("create table t1 using meters tags (1)") + sql = 'insert into t1 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i , i % 47) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t1 values ' + tdSql.execute(sql) + tdSql.query('select top(c,1) from meters interval(10a)') + tdSql.checkData(0,1,9) + def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_twa_test2.py b/tests/pytest/functions/function_twa_test2.py index 1ffaa3c6285c8eade8c3735945ba2218646f5654..b20f14357e22d38162b5d69cc3ae643f700f8754 100644 --- a/tests/pytest/functions/function_twa_test2.py +++ b/tests/pytest/functions/function_twa_test2.py @@ -120,6 +120,35 @@ class TDTestCase: tdSql.checkData(2, 1, -1.5) tdSql.checkData(3, 1, -2) + #TD-2533 twa+interval with large records + tdSql.execute("create table t4(ts timestamp, c int)") + sql = 'insert into t4 values ' + for i in range(20000): + sql = sql + '(%d, %d)' % (self.ts + i * 500, i + 1) + if i % 2000 == 0: + tdSql.execute(sql) + sql = 'insert into t4 values ' + tdSql.execute(sql) + tdSql.query('select twa(c) from t4 interval(10s)') + tdSql.checkData(0,1,10.999) + + # Test case: https://jira.taosdata.com:18080/browse/TD-2624 + tdSql.execute("create database test keep 7300") + tdSql.execute("use test") + tdSql.execute("create table st(ts timestamp, k int)") + tdSql.execute("insert into st values('2011-01-02 18:42:45.326', -1)") + tdSql.execute("insert into st values('2020-07-30 17:44:06.283', 0)") + tdSql.execute("insert into st values('2020-07-30 17:44:19.578', 9999999)") + tdSql.execute("insert into st values('2020-07-30 17:46:06.417', NULL)") + tdSql.execute("insert into st values('2020-11-09 18:42:25.538', 0)") + tdSql.execute("insert into st values('2020-12-29 17:43:11.641', 0)") + tdSql.execute("insert into st values('2020-12-29 18:43:17.129', 0)") + tdSql.execute("insert into st values('2020-12-29 18:46:19.109', NULL)") + tdSql.execute("insert into st values('2021-01-03 18:40:40.065', 0)") + + tdSql.query("select twa(k),first(ts) as taos1 from st where k <50 interval(17s)") + tdSql.checkRows(6) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index f00fe40c6981eda7630eb6d6c25fc3ff1ca4a4a2..528316700d184171641f7f686a3c12102b6c1feb 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -5,13 +5,24 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null & -./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4 -pidof taosd|xargs kill +IN_TDINTERNAL="community" +TDIR=`pwd` +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../../.. +fi +TOP_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep -v community|head -n1` +nohup $TAOSD_DIR >/dev/null & +cd - +./crash_gen.sh --valgrind -p -t 10 -s 350 -b 4 +pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` do +memError=(${memError//,/}) if [ -n "$memError" ]; then if [ "$memError" -gt 12 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ @@ -23,12 +34,16 @@ done grep 'start to execute\|definitely lost:' valgrind.err|grep -v 'grep'|uniq|tee crash_gen-definitely-lost-out.log for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | awk '{print $7}'` do - +defiMemError=(${defiMemError//,/}) if [ -n "$defiMemError" ]; then - if [ "$defiMemError" -gt 3 ]; then + if [ "$defiMemError" -gt 3 -a "$defiMemError" -lt 1013 ]; then echo -e "${RED} ## Memory errors number valgrind reports \ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" - exit 8 + exit 8 + elif [ "$defiMemError" -gt 1013 ];then #add for azure + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 fi fi -done \ No newline at end of file +done diff --git a/tests/pytest/import_merge/importCacheFileH.py b/tests/pytest/import_merge/importCacheFileH.py index 3398f7bdad8d7524e223f0a2847b579081686476..33724dfa6889903a34836844271f5843c7cb36e4 100644 --- a/tests/pytest/import_merge/importCacheFileH.py +++ b/tests/pytest/import_merge/importCacheFileH.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHO.py b/tests/pytest/import_merge/importCacheFileHO.py index 19520dc3d048a267c03850b608d47db20394c43d..9f552978f9bb5b0be72eb7cba7ebd073114e740b 100644 --- a/tests/pytest/import_merge/importCacheFileHO.py +++ b/tests/pytest/import_merge/importCacheFileHO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileHPO.py b/tests/pytest/import_merge/importCacheFileHPO.py index 9e0a57fb462cdb6ae1dee3c8bfee4488c99a76a0..f2d0a3ddbfb4c0a2a111aab423e32c0ed4683e47 100644 --- a/tests/pytest/import_merge/importCacheFileHPO.py +++ b/tests/pytest/import_merge/importCacheFileHPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileS.py b/tests/pytest/import_merge/importCacheFileS.py index 2f0af569e564b64cbb88c048762e8d41dcf21cbd..250952b2377f9d164c16801b6ead5f695d4f7c43 100644 --- a/tests/pytest/import_merge/importCacheFileS.py +++ b/tests/pytest/import_merge/importCacheFileS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileSub.py b/tests/pytest/import_merge/importCacheFileSub.py index 300bb6e8d09f4484e22e64a0f2caf50cab7c8ab4..663bdcd6351c02a91da309502e0a7f5ec9878086 100644 --- a/tests/pytest/import_merge/importCacheFileSub.py +++ b/tests/pytest/import_merge/importCacheFileSub.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileT.py b/tests/pytest/import_merge/importCacheFileT.py index ab33cf6a93c4bbd7f27967f6a70b61fc62230f65..f2feeeacff11a4c8a2d65feaa08cd1b57264e0c9 100644 --- a/tests/pytest/import_merge/importCacheFileT.py +++ b/tests/pytest/import_merge/importCacheFileT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTO.py b/tests/pytest/import_merge/importCacheFileTO.py index 00e22da976618b4ee870a5702542f669693e3727..421ca6a755425690e7ec76d9a932ce11a65cb141 100644 --- a/tests/pytest/import_merge/importCacheFileTO.py +++ b/tests/pytest/import_merge/importCacheFileTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importCacheFileTPO.py b/tests/pytest/import_merge/importCacheFileTPO.py index c6089e1d687fea6ac900931a8171e45f4923f0cb..ca582d7d5aad8106d7c3d12439001fda84d5004c 100644 --- a/tests/pytest/import_merge/importCacheFileTPO.py +++ b/tests/pytest/import_merge/importCacheFileTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataH2.py b/tests/pytest/import_merge/importDataH2.py index a21f0c47be2eb11a57182d22b56d4447ff2d7ab6..4de567d943a1fe843ec16dd4589054ebcef76488 100644 --- a/tests/pytest/import_merge/importDataH2.py +++ b/tests/pytest/import_merge/importDataH2.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO.py b/tests/pytest/import_merge/importDataHO.py index fdcaedd83c99482914805cb65a067cbcbc78a128..f40de71d6d3967aac6d851deae2810aed63bc3c8 100644 --- a/tests/pytest/import_merge/importDataHO.py +++ b/tests/pytest/import_merge/importDataHO.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHO2.py b/tests/pytest/import_merge/importDataHO2.py index b094701132677d6e72e834b16b7edc106682be14..518975e0b727db0d51bd6f5b68e690356988e2fe 100644 --- a/tests/pytest/import_merge/importDataHO2.py +++ b/tests/pytest/import_merge/importDataHO2.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataHPO.py b/tests/pytest/import_merge/importDataHPO.py index 9d74c0c3522eba3687618132c27ce95f86f91b91..c7f0625d4bf09de49066c9ea19d4936bdd00e811 100644 --- a/tests/pytest/import_merge/importDataHPO.py +++ b/tests/pytest/import_merge/importDataHPO.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastH.py b/tests/pytest/import_merge/importDataLastH.py index c8e5f62423dc21254d215fac41926aeb9ce5ba40..f736cb925a405c8588d01faad0b08759127962b4 100644 --- a/tests/pytest/import_merge/importDataLastH.py +++ b/tests/pytest/import_merge/importDataLastH.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHO.py b/tests/pytest/import_merge/importDataLastHO.py index 33215997a4220b65228b630ebc350bf9dae15046..b1eebfdd47ed1fbb5185b7547927ea50a29bc64e 100644 --- a/tests/pytest/import_merge/importDataLastHO.py +++ b/tests/pytest/import_merge/importDataLastHO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastHPO.py b/tests/pytest/import_merge/importDataLastHPO.py index fa8542f35b8e9639db20dc2e1886fca36140e825..9aa8127f50799039ceaa48dee957cdb903b890f0 100644 --- a/tests/pytest/import_merge/importDataLastHPO.py +++ b/tests/pytest/import_merge/importDataLastHPO.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastS.py b/tests/pytest/import_merge/importDataLastS.py index 2f595fef542955e616ef2e6ebcce7f23bce71d80..2c1559d2903a4fde104603bb2ad5945ea8eddd1e 100644 --- a/tests/pytest/import_merge/importDataLastS.py +++ b/tests/pytest/import_merge/importDataLastS.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastSub.py b/tests/pytest/import_merge/importDataLastSub.py index bfcad2d252cccb2404b3989c474310e0a19afe2e..415d1fcba27921267832323e6c3177678a215c9b 100644 --- a/tests/pytest/import_merge/importDataLastSub.py +++ b/tests/pytest/import_merge/importDataLastSub.py @@ -32,7 +32,7 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.deploy(1) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdSql.execute('reset query cache') tdSql.execute('drop database if exists db') @@ -60,9 +60,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 10 data totally repetitive") diff --git a/tests/pytest/import_merge/importDataLastT.py b/tests/pytest/import_merge/importDataLastT.py index 08e944eb6836a6f7f938639b40a6124f7bb64ec6..839320cc22352f1009d0d786aa58cf8827fc8ff6 100644 --- a/tests/pytest/import_merge/importDataLastT.py +++ b/tests/pytest/import_merge/importDataLastT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTO.py b/tests/pytest/import_merge/importDataLastTO.py index a82c0541414298accfdfb447c318ffafd88e5852..0c35519fcf76e7d2faab3a2cb2c4cc702fe93f0b 100644 --- a/tests/pytest/import_merge/importDataLastTO.py +++ b/tests/pytest/import_merge/importDataLastTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataLastTPO.py b/tests/pytest/import_merge/importDataLastTPO.py index ff75a1b2ae87891853fa93f0f49fff339cf6e488..c7864ae14d68a3ef8860125920b3286921f65356 100644 --- a/tests/pytest/import_merge/importDataLastTPO.py +++ b/tests/pytest/import_merge/importDataLastTPO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataS.py b/tests/pytest/import_merge/importDataS.py index 37627e8d6bcc8319805275525ef7874132e0da4d..f0d56a5016fe5b059e1b51719b799d801219c2ca 100644 --- a/tests/pytest/import_merge/importDataS.py +++ b/tests/pytest/import_merge/importDataS.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataSub.py b/tests/pytest/import_merge/importDataSub.py index 17e2b141b7014fd2ce4d31f8c335e122aca730ba..1643cc89155507be6c5fa2c072755a8439b21aab 100644 --- a/tests/pytest/import_merge/importDataSub.py +++ b/tests/pytest/import_merge/importDataSub.py @@ -60,7 +60,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataT.py b/tests/pytest/import_merge/importDataT.py index b0b7b82b790e81c475c714ba570a9a6f1801cc13..c23ad9a60533d94d34d11c70ea2743a1b8ab5c01 100644 --- a/tests/pytest/import_merge/importDataT.py +++ b/tests/pytest/import_merge/importDataT.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTO.py b/tests/pytest/import_merge/importDataTO.py index c0b57136af7a51c1d184eb5789d95efc0148118c..b4c0d1a8adb6d4a3751d5a4a6547d0d03eab793b 100644 --- a/tests/pytest/import_merge/importDataTO.py +++ b/tests/pytest/import_merge/importDataTO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importDataTPO.py b/tests/pytest/import_merge/importDataTPO.py index 8a1c9264b433484e768cdfffbccdb0966e3d15cd..38e8cfe9096135f1f9301d89bc8c6269d045e78f 100644 --- a/tests/pytest/import_merge/importDataTPO.py +++ b/tests/pytest/import_merge/importDataTPO.py @@ -57,9 +57,9 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdLog.info("================= step5") tdLog.info("import 20 data later with partly overlap") diff --git a/tests/pytest/import_merge/importHORestart.py b/tests/pytest/import_merge/importHORestart.py index f74c4c76d616753f8cadc87ba715e473eae0fb46..05b5b42afff14c695690b7651bd385ee78f3bafb 100644 --- a/tests/pytest/import_merge/importHORestart.py +++ b/tests/pytest/import_merge/importHORestart.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHPORestart.py b/tests/pytest/import_merge/importHPORestart.py index e5f79fbe6c2eb686d63f5b7de81651a98072a986..f9167709130d67b2909d3e9efd1acc482638836d 100644 --- a/tests/pytest/import_merge/importHPORestart.py +++ b/tests/pytest/import_merge/importHPORestart.py @@ -62,7 +62,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importHRestart.py b/tests/pytest/import_merge/importHRestart.py index be67039789dd660ddb0be8d56753737010e47e79..c1d50378b20a11b4d562b408d7bec07364969d19 100644 --- a/tests/pytest/import_merge/importHRestart.py +++ b/tests/pytest/import_merge/importHRestart.py @@ -54,7 +54,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importInsertThenImport.py b/tests/pytest/import_merge/importInsertThenImport.py index 292fae8c47b398dcbacdb8b76592826bcc8b6019..1372177de36c1f67ab85ba4a63c26553ee4a7de4 100644 --- a/tests/pytest/import_merge/importInsertThenImport.py +++ b/tests/pytest/import_merge/importInsertThenImport.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.stop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdLog.info("import 100 sequential data again") diff --git a/tests/pytest/import_merge/importLastH.py b/tests/pytest/import_merge/importLastH.py index 17fa233e3723e1d9e6cc4ec97c8eac2716751f00..2887c71e040b7ea89e42da69e597bcb843a2df5c 100644 --- a/tests/pytest/import_merge/importLastH.py +++ b/tests/pytest/import_merge/importLastH.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHO.py b/tests/pytest/import_merge/importLastHO.py index adb44fc0eab9a4923b0896a1357b24603f1ff3fd..1681212b6b8c5d002e6f3d2b5853534382bb4d10 100644 --- a/tests/pytest/import_merge/importLastHO.py +++ b/tests/pytest/import_merge/importLastHO.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastHPO.py b/tests/pytest/import_merge/importLastHPO.py index d8ed2d9ef10dbaa52e92ba50625be25c495091ea..389b3c11ed04b0359564a41c7b383e37e9bcabdd 100644 --- a/tests/pytest/import_merge/importLastHPO.py +++ b/tests/pytest/import_merge/importLastHPO.py @@ -55,7 +55,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastS.py b/tests/pytest/import_merge/importLastS.py index bf222a0d5f3c27c5c7774b384799a6a0aa5e4114..e6393bbfae2f41efd1ff740f4e542715f23f8adf 100644 --- a/tests/pytest/import_merge/importLastS.py +++ b/tests/pytest/import_merge/importLastS.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastSub.py b/tests/pytest/import_merge/importLastSub.py index 5a6b9f41502970149257b1aef5c7a37824401768..4ff6cf27bdf582040360e40ce13d28df7dfc8949 100644 --- a/tests/pytest/import_merge/importLastSub.py +++ b/tests/pytest/import_merge/importLastSub.py @@ -53,7 +53,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastT.py b/tests/pytest/import_merge/importLastT.py index 2b1be1fe2b37aa5e4d59bb6ecef7d0bb2180cf9d..145b1bd69097804728020ea1b3b8a0ccf7e86d27 100644 --- a/tests/pytest/import_merge/importLastT.py +++ b/tests/pytest/import_merge/importLastT.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTO.py b/tests/pytest/import_merge/importLastTO.py index ce189f63718956991982bd33296eb71bd5a82d45..a159dac4acea9c1c278025e157ff3fe09487e858 100644 --- a/tests/pytest/import_merge/importLastTO.py +++ b/tests/pytest/import_merge/importLastTO.py @@ -57,7 +57,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importLastTPO.py b/tests/pytest/import_merge/importLastTPO.py index 627d090855ac320f56a90fb647101e8ccc315acd..453b922a68bc7a051aa60d51943ffb759444ee5b 100644 --- a/tests/pytest/import_merge/importLastTPO.py +++ b/tests/pytest/import_merge/importLastTPO.py @@ -59,7 +59,7 @@ class TDTestCase: tdLog.info("================= step4") tdDnodes.stop(1) - tdLog.sleep(5) + #tdLog.sleep(5) tdDnodes.start(1) tdLog.info("================= step5") diff --git a/tests/pytest/import_merge/importSRestart.py b/tests/pytest/import_merge/importSRestart.py index 29f5a19b54fb0cbf1eb81a66c66813f3fcdd9982..58f0f60e0af0660e4550966c2f0b1741aefbc33c 100644 --- a/tests/pytest/import_merge/importSRestart.py +++ b/tests/pytest/import_merge/importSRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importSubRestart.py b/tests/pytest/import_merge/importSubRestart.py index b1a6f30c436c5aa52d5ee4affa03d18c5475ced7..85594e1772409f7c24a5d2ce8aa25b6ba7567a5d 100644 --- a/tests/pytest/import_merge/importSubRestart.py +++ b/tests/pytest/import_merge/importSubRestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTORestart.py b/tests/pytest/import_merge/importTORestart.py index 07eb6c28cb79c16504f2acab036a3b66c3bccabf..10ac77d759192de329ab0818b79610259d05b7d7 100644 --- a/tests/pytest/import_merge/importTORestart.py +++ b/tests/pytest/import_merge/importTORestart.py @@ -64,7 +64,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTPORestart.py b/tests/pytest/import_merge/importTPORestart.py index 10bbf3efced90b6c255326c0e8b8e0c106a3a038..ab86a0247f179bd6dab397a069ca9b4aaf4da61f 100644 --- a/tests/pytest/import_merge/importTPORestart.py +++ b/tests/pytest/import_merge/importTPORestart.py @@ -68,7 +68,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/import_merge/importTRestart.py b/tests/pytest/import_merge/importTRestart.py index 63a9368eca2b6a3377b32dc6a3948cf5a0ba1186..6106bdd75310369d53ce8148f937655301b64c12 100644 --- a/tests/pytest/import_merge/importTRestart.py +++ b/tests/pytest/import_merge/importTRestart.py @@ -61,7 +61,7 @@ class TDTestCase: tdLog.info("================= step5") tdDnodes.forcestop(1) tdDnodes.start(1) - tdLog.sleep(10) + #tdLog.sleep(10) tdLog.info("================= step6") tdSql.query('select * from tb1') diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index da797f788fcc3cc1ae06843b70da9aaf7230e97c..6489fd31ffd6f595d15d996b92f5c63fc7e2b8a9 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -38,7 +38,7 @@ class RestfulInsert: for i in range(loop): tableID = threadID * tablesPerThread if tableID + i >= self.numOfTables : break - name = 'beijing' if tableID % 2 == 0 else 'shanghai' + name = 'beijing' if (tableID + i) % 2 == 0 else 'shanghai' data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) response = requests.post(self.url, data, headers = self.header) if response.status_code != 200: diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index 05d6ec1cecdf59737f9bf5a8b2417b9adde0cc87..ad26c480048e04e8a0955f908c0396d63726513e 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -19,7 +19,8 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py -python3 ./test.py -f query/isNullTest.py +#python3 ./test.py -f insert/before_1970.py +python3 bug2265.py #table python3 ./test.py -f table/alter_wal0.py @@ -33,7 +34,7 @@ python3 ./test.py -f table/alter_column.py python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py -python3 ./test.py -f table/queryWithTaosdKilled.py + # tag python3 ./test.py -f tag_lite/filter.py @@ -163,19 +164,29 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py -python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2118.py +python3 ./test.py -f query/bug2143.py +python3 ./test.py -f query/sliding.py +python3 ./test.py -f query/unionAllTest.py python3 ./test.py -f query/bug2281.py python3 ./test.py -f query/bug2119.py -python3 bug2265.py +python3 ./test.py -f query/isNullTest.py +python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/floatCompare.py #stream python3 ./test.py -f stream/metric_1.py +python3 ./test.py -f stream/metric_n.py python3 ./test.py -f stream/new.py python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py #python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py +python3 ./test.py -f stream/sys.py +python3 ./test.py -f stream/table_1.py +python3 ./test.py -f stream/table_n.py #alter table python3 ./test.py -f alter/alter_table_crash.py @@ -184,6 +195,7 @@ python3 ./test.py -f alter/alter_table_crash.py python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py +python3 ./test.py -f client/noConnectionErrorTest.py # Misc python3 testCompress.py @@ -207,7 +219,7 @@ python3 ./test.py -f functions/function_spread.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 -#python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py @@ -219,10 +231,10 @@ python3 test.py -f query/queryFillTest.py python3 test.py -f tools/taosdemoTest.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/lowaTest.py +python3 test.py -f tools/taosdemoTest2.py # subscribe python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py +#python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py - diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh index fededea3bb34481aa34c37d4a802ba183494e491..4ec517a0bf1c5eff8ad670cf28ab63d5ce818460 100755 --- a/tests/pytest/pytest_2.sh +++ b/tests/pytest/pytest_2.sh @@ -12,6 +12,10 @@ python3 ./test.py -f update/merge_commit_data2.py python3 ./test.py -f update/merge_commit_data2_update0.py python3 ./test.py -f update/merge_commit_last-0.py python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py # wal -python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file +python3 ./test.py -f wal/addOldWalTest.py + +# function +python3 ./test.py -f functions/all_null_value.py \ No newline at end of file diff --git a/tests/pytest/query/last_cache.py b/tests/pytest/query/last_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..c31d9821e2695dd7883674519325c24f0e473bf2 --- /dev/null +++ b/tests/pytest/query/last_cache.py @@ -0,0 +1,133 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.tables = 10 + self.rows = 20 + self.perfix = 't' + self.ts = 1601481600000 + + def insertData(self): + print("==============step1") + tdSql.execute("create table st (ts timestamp, c1 int) tags(t1 int)") + + for i in range(self.tables): + tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) + for j in range(self.rows): + tc = self.ts + j * 60000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, i, tc, j)) + + def executeQueries(self): + print("==============step2") + tdSql.query("select last(c1) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) from %s%d where ts <= %d" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) as b from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) from %s%d interval(1m)" % (self.perfix, 1)) + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from %s%d interval(1d)" % (self.perfix, 1)) + tdSql.checkData(0, 1, 19) + + tdSql.query("select last(c1) from %s%d where ts <= %d interval(1m)" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkRows(5) + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from st") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) as c from st where t1 <= 5") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where t1 <= 5 and ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last(c1) from st interval(1m)") + tdSql.checkData(1, 1, 1) + + tdSql.query("select last(c1) from st interval(1d)") + tdSql.checkData(0, 1, 19) + + tdSql.query("select last(c1) from st group by t1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last(c1) as c from st where ts <= %d interval(1m) group by t1" % (self.ts + 4 * 60000)) + tdSql.checkData(1, 1, 1) + tdSql.checkRows(50) + + def run(self): + tdSql.prepare() + + # last_cache_0.sim + tdSql.execute("create database test1 cachelast 0") + tdSql.execute("use test1") + self.insertData() + self.executeQueries() + + tdSql.execute("alter database test1 cachelast 1") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + tdSql.execute("alter database test1 cachelast 0") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + # last_cache_1.sim + tdSql.execute("create database test2 cachelast 1") + tdSql.execute("use test2") + self.insertData() + self.executeQueries() + + tdSql.execute("alter database test2 cachelast 0") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + tdSql.execute("alter database test2 cachelast 1") + self.executeQueries() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..d9e09dae7acdcd8c7401f1370da9f544c850872d --- /dev/null +++ b/tests/pytest/query/last_row_cache.py @@ -0,0 +1,186 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.tables = 10 + self.rows = 20 + self.perfix = 't' + self.ts = 1601481600000 + + def insertData(self): + print("==============step1") + tdSql.execute("create table st (ts timestamp, c1 int) tags(t1 int)") + + for i in range(self.tables): + tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) + for j in range(self.rows): + tc = self.ts + j * 60000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, i, tc, j)) + + def executeQueries(self): + print("==============step2") + tdSql.query("select last_row(c1) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) from %s%d where ts <= %d" % (self.perfix, 1, self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as b from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) from st") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) as c from st where ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as c from st where t1 < 5") + tdSql.checkData(0, 0, 19) + + tdSql.query("select last_row(c1) as c from st where t1 <= 5 and ts <= %d" % (self.ts + 4 * 60000)) + tdSql.checkData(0, 0, 4) + + tdSql.query("select last_row(c1) as c from st group by t1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 19) + + tc = self.ts + 1 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + + tc = self.ts + 3 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + tc = self.ts + 5 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + + tc = self.ts + 7 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + def insertData2(self): + tc = self.ts + 1 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + + tc = self.ts + 3 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + tc = self.ts + 5 * 3600000 + tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + + tc = self.ts + 7 * 3600000 + tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + + def executeQueries2(self): + # For stable + tc = self.ts + 6 * 3600000 + tdSql.query("select last_row(c1) from st where ts < %d " % tc) + tdSql.checkData(0, 0, -1) + + tc = self.ts + 8 * 3600000 + tdSql.query("select last_row(*) from st where ts < %d " % tc) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(*) from st") + tdSql.checkData(0, 1, None) + + tc = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts < %d " % tc) + tdSql.checkData(0, 1, None) + + tc1 = self.ts + 1 * 3600000 + tc2 = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts > %d and ts <= %d" % (tc1, tc2)) + tdSql.checkData(0, 1, None) + + # For table + tc = self.ts + 6 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, -1) + + tc = self.ts + 8 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(*) from %s%d" % (self.perfix, 1)) + tdSql.checkData(0, 1, None) + + tc = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from %s%d where ts <= %d" % (self.perfix, 1, tc)) + tdSql.checkData(0, 1, None) + + tc1 = self.ts + 1 * 3600000 + tc2 = self.ts + 4 * 3600000 + tdSql.query("select last_row(*) from st where ts > %d and ts <= %d" % (tc1, tc2)) + tdSql.checkData(0, 1, None) + + def run(self): + tdSql.prepare() + + print("============== last_row_cache_0.sim") + tdSql.execute("create database test1 cachelast 0") + tdSql.execute("use test1") + self.insertData() + self.executeQueries() + self.insertData2() + self.executeQueries2() + + print("============== alter last cache") + tdSql.execute("alter database test1 cachelast 1") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test1 cachelast 0") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + print("============== last_row_cache_1.sim") + tdSql.execute("create database test2 cachelast 1") + tdSql.execute("use test2") + self.insertData() + self.executeQueries() + self.insertData2() + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 0") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 1") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index da0ef96d0efde77b716ded06cc7efaa96ea3bb7b..87635f86f39fb317907f689307eeefa9f3aef124 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -16,7 +16,7 @@ import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql - +from util.dnodes import tdDnodes class TDTestCase: def init(self, conn, logSql): @@ -44,6 +44,25 @@ class TDTestCase: tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'") tdSql.checkRows(1) + ## test case for https://jira.taosdata.com:18080/browse/TD-2488 + tdSql.execute("create table m1(ts timestamp, k int) tags(a int)") + tdSql.execute("create table t1 using m1 tags(1)") + tdSql.execute("create table t2 using m1 tags(2)") + tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)") + tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)") + tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)") + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 871c076c0813f0ccd340d33bff992401e522df4a..9cc468b34e6133af393d9d0d800e64d14e85c430 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -24,7 +24,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.ts = 1593548685000 + self.ts = 1593548685000 def run(self): tdSql.prepare() @@ -84,6 +84,22 @@ class TDTestCase: tdDnodes.start(1) tdSql.query("select last(*) from t interval(1s)") tdSql.checkRows(10000) + + # test case for https://jira.taosdata.com:18080/browse/TD-2601 + newTs = 1601481600000 + + tdSql.execute("create database test2") + tdSql.execute("use test2") + tdSql.execute("create table t (ts timestamp, voltage int)") + for i in range(100): + tdSql.execute("insert into t values(%d, %d)" % (newTs + i * 10000000, i)) + + tdSql.query("select sum(voltage) from t where ts >='2020-10-01 00:00:00' and ts <='2020-12-01 00:00:00' interval(1n) fill(NULL)") + tdSql.checkRows(3) + tdSql.checkData(0, 1, 4950) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, None) + def stop(self): diff --git a/tests/pytest/stream/metric_n.py b/tests/pytest/stream/metric_n.py new file mode 100644 index 0000000000000000000000000000000000000000..d223fe81fc79835047bac8ca2341cdbeac2e6617 --- /dev/null +++ b/tests/pytest/stream/metric_n.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tbNum = 10 + rowNum = 20 + totalNum = tbNum * rowNum + + tdSql.prepare() + + tdLog.info("===== preparing data =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step 1 =====") + tdSql.query("select count(*), count(tbcol), count(tbcol2) from stb interval(1d)") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 2 =====") + tdSql.execute("create table strm_c3 as select count(*), count(tbcol), count(tbcol2) from stb interval(1d)") + + tdLog.info("===== step 3 =====") + tdSql.execute("create table strm_c32 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + + tdLog.info("===== step 4 =====") + tdSql.query("select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 5 =====") + tdSql.execute("create table strm_c31 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from stb interval(1d)") + + tdLog.info("===== step 6 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.execute("create table strm_avg as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb interval(1d)") + + tdLog.info("===== step 7 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from stb where ts < now + 4m interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, totalNum) + + tdLog.info("===== step 8 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from stb where ts < now + 4m interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, totalNum) + + tdLog.info("===== step 9 =====") + tdSql.waitedQuery("select * from strm_c3", 1, 120) + tdSql.checkData(0, 1, totalNum) + tdSql.checkData(0, 2, totalNum) + tdSql.checkData(0, 3, totalNum) + + tdLog.info("===== step 10 =====") + tdSql.waitedQuery("select * from strm_c31", 1, 30) + for i in range(1, 10): + tdSql.checkData(0, i, totalNum) + + tdLog.info("===== step 11 =====") + tdSql.waitedQuery("select * from strm_avg", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 1900) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/sys.py b/tests/pytest/stream/sys.py new file mode 100644 index 0000000000000000000000000000000000000000..a73e7043e8c65b2eb9c78fbcb99d4e546ddf9ae4 --- /dev/null +++ b/tests/pytest/stream/sys.py @@ -0,0 +1,62 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# migrated from 'stream_on_sys.sim' +# -*- coding: utf-8 -*- +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + updatecfgDict = {'monitor': 1} + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def run(self): + time.sleep(5) + tdSql.execute("use log") + + tdSql.execute("create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s)") + tdSql.execute("create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s)") + tdSql.execute("create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s)") + tdSql.execute("create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s)") + tdSql.execute("create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s)") + tdSql.execute("create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s)") + + sqls = [ + "select * from cpustrm", + "select * from memstrm", + "select * from diskstrm", + "select * from bandstrm", + "select * from reqstrm", + "select * from iostrm", + ] + for sql in sqls: + (rows, _) = tdSql.waitedQuery(sql, 1, 120) + if rows < 1: + tdLog.exit("failed: sql:%s, expect at least one row" % sql) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + diff --git a/tests/pytest/stream/table_1.py b/tests/pytest/stream/table_1.py new file mode 100644 index 0000000000000000000000000000000000000000..a9fd57393112177fe0e290cadeaabd97d669daca --- /dev/null +++ b/tests/pytest/stream/table_1.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def createFuncStream(self, expr, suffix, value): + tbname = "strm_" + suffix + tdLog.info("create stream table %s" % tbname) + tdSql.query("select %s from tb1 interval(1d)" % expr) + tdSql.checkData(0, 1, value) + tdSql.execute("create table %s as select %s from tb1 interval(1d)" % (tbname, expr)) + + def checkStreamData(self, suffix, value): + sql = "select * from strm_" + suffix + tdSql.waitedQuery(sql, 1, 120) + tdSql.checkData(0, 1, value) + + def run(self): + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + self.createFuncStream("count(*)", "c1", rowNum) + self.createFuncStream("count(tbcol)", "c2", rowNum) + self.createFuncStream("count(tbcol2)", "c3", rowNum) + self.createFuncStream("avg(tbcol)", "av", 9.5) + self.createFuncStream("sum(tbcol)", "su", 190) + self.createFuncStream("min(tbcol)", "mi", 0) + self.createFuncStream("max(tbcol)", "ma", 19) + self.createFuncStream("first(tbcol)", "fi", 0) + self.createFuncStream("last(tbcol)", "la", 19) + self.createFuncStream("stddev(tbcol)", "st", 5.766281297335398) + self.createFuncStream("percentile(tbcol, 1)", "pe", 0.19) + self.createFuncStream("count(tbcol)", "as", rowNum) + + self.checkStreamData("c1", rowNum) + self.checkStreamData("c2", rowNum) + self.checkStreamData("c3", rowNum) + self.checkStreamData("av", 9.5) + self.checkStreamData("su", 190) + self.checkStreamData("mi", 0) + self.checkStreamData("ma", 19) + self.checkStreamData("fi", 0) + self.checkStreamData("la", 19) + self.checkStreamData("st", 5.766281297335398) + self.checkStreamData("pe", 0.19) + self.checkStreamData("as", rowNum) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/stream/table_n.py b/tests/pytest/stream/table_n.py new file mode 100644 index 0000000000000000000000000000000000000000..371af769778bce1eb1e6cf1bac89333006c582a8 --- /dev/null +++ b/tests/pytest/stream/table_n.py @@ -0,0 +1,143 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== preparing data =====") + tdSql.execute( + "create table stb(ts timestamp, tbcol int, tbcol2 float) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (now - %dm, %d, %d)" % + (i, 1440 - j, j, j)) + time.sleep(0.1) + + tdLog.info("===== step 1 =====") + tdSql.query("select count(*), count(tbcol), count(tbcol2) from tb1 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 2 =====") + tdSql.execute("create table strm_c3 as select count(*), count(tbcol), count(tbcol2) from tb1 interval(1d)") + + tdLog.info("===== step 3 =====") + tdSql.execute("create table strm_c32 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + + tdLog.info("===== step 4 =====") + tdSql.query("select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 5 =====") + tdSql.execute("create table strm_c31 as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from tb1 interval(1d)") + + tdLog.info("===== step 6 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from tb1 interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.execute("create table strm_avg as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from tb1 interval(1d)") + + tdLog.info("===== step 7 =====") + tdSql.query("select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from tb1 interval(1d)") + tdSql.checkData(0, 1, 5.766281297335398) + tdSql.checkData(0, 3, 0.19) + tdSql.execute("create table strm_ot as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from tb1 interval(1d)") + + tdLog.info("===== step 8 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 interval(1d)") + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, 5.766281297335398) + tdSql.checkData(0, 8, 0.19) + tdSql.checkData(0, 9, rowNum) + tdSql.execute("create table strm_to as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 interval(1d)") + + tdLog.info("===== step 9 =====") + tdSql.query("select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 where ts < now + 4m interval(1d)") + tdSql.checkData(0, 9, rowNum) + tdSql.execute("create table strm_wh as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from tb1 where ts < now + 4m interval(1d)") + + tdLog.info("===== step 10 =====") + tdSql.waitedQuery("select * from strm_c3", 1, 120) + tdSql.checkData(0, 1, rowNum) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + + tdLog.info("===== step 11 =====") + tdSql.waitedQuery("select * from strm_c31", 1, 30) + for i in range(1, 10): + tdSql.checkData(0, i, rowNum) + + tdLog.info("===== step 12 =====") + tdSql.waitedQuery("select * from strm_avg", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + + tdLog.info("===== step 13 =====") + tdSql.waitedQuery("select * from strm_ot", 1, 20) + tdSql.checkData(0, 1, 5.766281297335398) + tdSql.checkData(0, 3, 0.19) + + tdLog.info("===== step 14 =====") + tdSql.waitedQuery("select * from strm_to", 1, 20) + tdSql.checkData(0, 1, 9.5) + tdSql.checkData(0, 2, 190) + tdSql.checkData(0, 3, 0) + tdSql.checkData(0, 4, 19) + tdSql.checkData(0, 5, 0) + tdSql.checkData(0, 6, 19) + tdSql.checkData(0, 7, 5.766281297335398) + tdSql.checkData(0, 8, 0.19) + tdSql.checkData(0, 9, rowNum) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/alter_wal0.py b/tests/pytest/table/alter_wal0.py index 15ad69998f450b8e385cbf58052d246d9de27380..807dd316f2f5b154817217c9cf28821242bc9176 100644 --- a/tests/pytest/table/alter_wal0.py +++ b/tests/pytest/table/alter_wal0.py @@ -43,7 +43,8 @@ class TDTestCase: print("==============step2") tdDnodes.stopAll() - filename = '/var/lib/taos/mnode/wal/wal0' + path = tdDnodes.getDnodesRootDir() + filename = path + '/dnode1/data/mnode/wal/wal0' with open(filename, 'rb') as f1: temp = f1.read() @@ -57,12 +58,12 @@ class TDTestCase: print("==============step3") tdSql.execute("use demo;") tdSql.query('show tables;') - tdSql.checkRows(10) + tdSql.checkRows(9) for i in range(11,21): tdSql.execute("CREATE table if not exists test{num} using meters tags({num});".format(num=i)) tdSql.query('show tables;') - tdSql.checkRows(20) + tdSql.checkRows(19) print("==============check table numbers and create 10 tables") diff --git a/tests/pytest/table/create.py b/tests/pytest/table/create.py index 8fedd4e92046f58a94db66a1e696edfe5d253982..a0991d674a23065cbec5ae3b69d6bdf37ee363ac 100644 --- a/tests/pytest/table/create.py +++ b/tests/pytest/table/create.py @@ -39,6 +39,23 @@ class TDTestCase: except Exception as e: tdLog.exit(e) + # case for defect: https://jira.taosdata.com:18080/browse/TD-2560 + tdSql.execute("create table db.tb02 using st tags(2)") + tdSql.execute("create table db.tb03 using st tags(3)") + tdSql.execute("create table db.tb04 using st tags(4)") + + tdSql.query("show tables like 'tb%' ") + tdSql.checkRows(4) + + tdSql.query("show tables like 'tb0%' ") + tdSql.checkRows(3) + + tdSql.execute("create table db.st0 (ts timestamp, i int) tags(j int)") + tdSql.execute("create table db.st1 (ts timestamp, i int, c2 int) tags(j int, loc nchar(20))") + + tdSql.query("show stables like 'st%' ") + tdSql.checkRows(3) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tools/lowaTest.py b/tests/pytest/tools/lowaTest.py index 523229dd463d54c5b2cd23a9a3d4d547858a3b5c..2b65dcf3eff1ed9ed7275fd774807cfa0318ec81 100644 --- a/tests/pytest/tools/lowaTest.py +++ b/tests/pytest/tools/lowaTest.py @@ -51,7 +51,7 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - os.system("yes | %slowa -f tools/insert.json" % binPath) + os.system("yes | %staosdemox -f tools/insert.json" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 6b6296e61a7965facd3a414ad635a4c64af43cc4..28f451b6a08a43508352c75b0c498251c9afca54 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -22,12 +22,15 @@ import argparse import os.path class taosdemoPerformace: - def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond): + def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay): self.commitID = commitID self.dbName = dbName self.createTableTime = createTableTime self.insertRecordsTime = insertRecordsTime - self.recordsPerSecond = recordsPerSecond + self.recordsPerSecond = recordsPerSecond + self.avgDelay = avgDelay + self.maxDelay = maxDelay + self.minDelay = minDelay self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -43,12 +46,15 @@ class taosdemoPerformace: cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("use %s" % self.dbName) - cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50))") + cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") print("==================== taosdemo performance ====================") print("create tables time: %f" % self.createTableTime) print("insert records time: %f" % self.insertRecordsTime) print("records per second: %f" % self.recordsPerSecond) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s')" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID)) + print("avg delay: %f" % self.avgDelay) + print("max delay: %f" % self.maxDelay) + print("min delay: %f" % self.minDelay) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) cursor.execute("drop database if exists taosdemo_insert_test") cursor.close() @@ -86,8 +92,28 @@ if __name__ == '__main__': action='store', type=float, help='records per request') + parser.add_argument( + '-avg', + '---avg-delay', + action='store', + type=float, + help='avg delay') + parser.add_argument( + '-max', + '---max-delay', + action='store', + type=float, + help='max delay') + parser.add_argument( + '-min', + '---min-delay', + action='store', + type=float, + help='min delay') + args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second) + perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, + args.avg_delay, args.max_delay, args.min_delay) perftest.createTablesAndStoreData() \ No newline at end of file diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 83cb4e8d9994b1729ee13dda6829de300d9e8d7a..afea23372f5bd13949bb6967392167df59c3d2b2 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -291,8 +291,8 @@ class TDDnode: break tdLog.debug("the dnode:%d has been started." % (self.index)) else: - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) + tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index)) + time.sleep(10) # time.sleep(5) diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 4dd3957f390237c6dddc6b1f66b14ee177b6dbcf..cde51ebdbfc90f50bbe8d7941fd4f697d8b59ee5 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -134,8 +134,6 @@ run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim #unsupport run general/parser/slimit_alter_tags.sim -#unsupport run general/parser/stream_on_sys.sim -#unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim run general/stable/metrics.sim @@ -213,9 +211,6 @@ run general/vector/table_time.sim run general/stream/restart_stream.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/general/db/alter_tables_d2.sim b/tests/script/general/db/alter_tables_d2.sim index 9ef39fb55664ad0f984a3a2b4971daa69feec074..cd3121057b2c8d02755e89c40e11ca7b9d52b4ce 100644 --- a/tests/script/general/db/alter_tables_d2.sim +++ b/tests/script/general/db/alter_tables_d2.sim @@ -11,11 +11,26 @@ system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start - -sleep 3000 sql connect sql create dnode $hostname2 -sleep 1000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ============================ step1 @@ -80,10 +95,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -110,7 +138,6 @@ if $rows != 40 then return -1 endi - sql insert into db.t100 values(now, 1) sql insert into db.t101 values(now, 1) sql insert into db.t102 values(now, 1) @@ -144,10 +171,24 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -198,7 +239,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -268,10 +309,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -321,7 +375,7 @@ if $rows != 80 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -351,13 +405,26 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache -sleep 1000 +sleep 100 sql show db.tables if $rows != 80 then @@ -401,10 +468,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +step1xx: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step1xx +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1xx +endi sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) @@ -454,7 +534,7 @@ if $rows != 100 then endi sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then diff --git a/tests/script/general/db/alter_tables_v1.sim b/tests/script/general/db/alter_tables_v1.sim index dde5eb6d9ecfbce4467a54abf883cc6b9ffbb696..20c4c7336312b7eb701fe04c6b4158f466e43146 100644 --- a/tests/script/general/db/alter_tables_v1.sim +++ b/tests/script/general/db/alter_tables_v1.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -51,9 +49,22 @@ print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step2 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step2 +endi sql create table db.t10 using db.st tags(0) sql create table db.t11 using db.st tags(1) @@ -91,9 +102,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 30 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t20 using db.st tags(0) sql create table db.t21 using db.st tags(1) @@ -124,7 +148,7 @@ sql insert into db.t29 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t0 if $rows != 1 then @@ -148,9 +172,22 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step7 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step7 +endi sql reset query cache sleep 1000 @@ -190,9 +227,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 40 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t30 using db.st tags(0) sql create table db.t31 using db.st tags(1) @@ -249,12 +299,24 @@ if $rows != 40 then return -1 endi - print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -294,9 +356,22 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 50 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi sql create table db.t40 using db.st tags(0) sql create table db.t41 using db.st tags(1) diff --git a/tests/script/general/db/alter_tables_v4.sim b/tests/script/general/db/alter_tables_v4.sim index 7c9262874d8f07cef92df2585fe60261ea763080..b57b2c03208274949a901cf72c2d580133ca1ecd 100644 --- a/tests/script/general/db/alter_tables_v4.sim +++ b/tests/script/general/db/alter_tables_v4.sim @@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect print ============================ step1 @@ -70,9 +68,22 @@ endi print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step3 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step3 +endi sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -131,9 +142,22 @@ endi print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step5 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step5 +endi sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) @@ -184,7 +208,7 @@ sql insert into db.t219 values(now, 1) print ============================ step6 sql reset query cache -sleep 1000 +sleep 100 sql select * from db.t000 if $rows != 1 then @@ -250,9 +274,22 @@ endi print ============================ step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step9 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step9 +endi sql create table db.t300 using db.st tags(0) sql create table db.t301 using db.st tags(1) @@ -331,9 +368,22 @@ endi print ============================ step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step10 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step10 +endi sql reset query cache sleep 1000 @@ -378,9 +428,24 @@ endi print ============================ step12 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25 -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +step12: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step12 +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step12 +endi + + sql create table db.t400 using db.st tags(0) sql create table db.t401 using db.st tags(1) sql create table db.t402 using db.st tags(2) diff --git a/tests/script/general/db/delete_reusevnode.sim b/tests/script/general/db/delete_reusevnode.sim index 5781c384fe9bfd9ac804a3cd85a3e88a1e4bc278..4364a760a6b61be89f00f5a22d19c9ecf9015cb2 100644 --- a/tests/script/general/db/delete_reusevnode.sim +++ b/tests/script/general/db/delete_reusevnode.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======== step1 @@ -38,7 +37,6 @@ if $rows != 0 then endi system sh/stop_dnodes.sh -sleep 3000 system sh/deploy.sh -n dnode1 -i 1 print ========= start dnodes diff --git a/tests/script/general/db/delete_writing1.sim b/tests/script/general/db/delete_writing1.sim index 93d41b2fa37190e86133c47b087bc0a6a2cb71f1..8b369b4e3d0fc918aad19d2295583a0e2c1fda59 100644 --- a/tests/script/general/db/delete_writing1.sim +++ b/tests/script/general/db/delete_writing1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database db diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 94ecb59f75304d99f48ebdb644be432370f86f2a..7e12d30ac91eb49025fbebed05336e15cc6900ac 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -81,7 +81,7 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"data":[],"rows":0}@ then return -1 endi diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index 773f92afcfd1fb126f137befd9de1107b050356f..df9a4598e0ba5438e45d299d467d86b2989eb825 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -106,7 +106,7 @@ while $x < 5000 endw system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 1000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 3505ad1a28139a85e62193eddeb050dfde0bbc0d..78b63c2baf460907a2d88836c52f157a8753c89c 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -244,3 +244,120 @@ if $data00 != -2.000000000 then print expect -2.000000000, actual: $data00 return -1 endi + +sql create table tm1 (ts timestamp, k int); +sql insert into tm1 values('2020-10-30 18:11:56.680', -1000); +sql insert into tm1 values('2020-11-19 18:11:45.773', NULL); +sql insert into tm1 values('2020-12-09 18:11:17.098', NULL); +sql insert into tm1 values('2020-12-20 18:11:49.412', 1); +sql insert into tm1 values('2020-12-23 18:11:50.412', 2); +sql insert into tm1 values('2020-12-28 18:11:52.412', 3); + +print =====================> td-2610 +sql select twa(k)from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-9 18:11:17.098' +if $rows != 0 then + return -1 +endi + +print =====================> td-2609 +sql select apercentile(k, 50) from tm1 where ts>='2020-10-30 18:11:56.680' and ts<='2020-12-09 18:11:17.098' +if $rows != 1 then + return -1 +endi + +if $data00 != -1000.000000000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 1000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect +sleep 500 + +sql use m_func_db0 + +print =====================> td-2583 +sql select min(k) from tm1 where ts>='2020-11-19 18:11:45.773' and ts<='2020-12-20 18:11:49.412' +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + print expect 1, actual: $data00 + return -1 +endi + +print =====================> td-2601 +sql select count(*) from tm1 where ts<='2020-6-1 00:00:00' and ts>='2020-1-1 00:00:00' interval(1n) fill(NULL) +if $rows != 0 then + return -1 +endi + +print =====================> td-2615 +sql select last(ts) from tm1 interval(17a) limit 776 offset 3 +if $rows != 3 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) limit 1000 offset 4 +if $rows != 2 then + return -1 +endi + +sql select last(ts) from tm1 interval(17a) order by ts desc limit 1000 offset 0 +if $rows != 6 then + return -1 +endi + +print ==================> td-2624 +sql create table tm2(ts timestamp, k int, b binary(12)); +sql insert into tm2 values('2011-01-02 18:42:45.326', -1,'abc'); +sql insert into tm2 values('2020-07-30 17:44:06.283', 0, null); +sql insert into tm2 values('2020-07-30 17:44:19.578', 9999999, null); +sql insert into tm2 values('2020-07-30 17:46:06.417', NULL, null); +sql insert into tm2 values('2020-11-09 18:42:25.538', 0, null); +sql insert into tm2 values('2020-12-29 17:43:11.641', 0, null); +sql insert into tm2 values('2020-12-29 18:43:17.129', 0, null); +sql insert into tm2 values('2020-12-29 18:46:19.109', NULL, null); +sql insert into tm2 values('2021-01-03 18:40:40.065', 0, null); + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s); +if $rows != 6 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data02 != @11-01-02 18:42:45.326@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi + +if $data21 != 0.000000000 then + return -1 +endi + +sql select twa(k),first(ts) from tm2 where k <50 interval(17s) order by ts desc; +if $rows != 6 then + return -1 +endi + +sql select twa(k),first(ts),count(k),first(k) from tm2 interval(17s) limit 20 offset 0; +if $rows != 9 then + return -1 +endi + +if $data00 != @11-01-02 18:42:42.000@ then + return -1 +endi + +if $data10 != @20-07-30 17:43:59.000@ then + return -1 +endi diff --git a/tests/script/general/parser/mixed_blocks.sim b/tests/script/general/parser/mixed_blocks.sim index d3558560df95d02256e0bc6b4a04ae9fae6e5890..946dbe8835dfcade3916754ebea12dd0d0c3a59d 100644 --- a/tests/script/general/parser/mixed_blocks.sim +++ b/tests/script/general/parser/mixed_blocks.sim @@ -144,4 +144,20 @@ if $data03 != 319 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print ===================> TD-2488 +sql create table m1(ts timestamp, k int) tags(a int); +sql create table t1 using m1 tags(1); +sql create table t2 using m1 tags(2); +sql insert into t1 values('2020-1-1 1:1:1', 1); +sql insert into t1 values('2020-1-1 1:10:1', 2); +sql insert into t2 values('2020-1-1 1:5:1', 99); + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql select ts from m1 where ts='2020-1-1 1:5:1' +if $rows != 1 then + return -1 +endi \ No newline at end of file diff --git a/tests/script/general/parser/repeatStream.sim b/tests/script/general/parser/repeatStream.sim deleted file mode 100644 index 616679e78bf8095e175ef91abf96c918626d6acb..0000000000000000000000000000000000000000 --- a/tests/script/general/parser/repeatStream.sim +++ /dev/null @@ -1,9 +0,0 @@ -$i = 1 -$repeats = 5 -while $i <= $repeats - print ====== repeat: $i - run general/parser/stream.sim - $i = $i + 1 -endw - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim deleted file mode 100644 index 1c8eb82c79a537ddbb322bdb164541263d92373c..0000000000000000000000000000000000000000 --- a/tests/script/general/parser/stream_on_sys.sim +++ /dev/null @@ -1,62 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 500 -sql connect -print ======================== stream_on_sys.sim - -$db = log -$tb = tb -$mt = mt -$strm = strm -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -$i = 0 - -sql use $db - -sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) -sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) -sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) -sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) -sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) -sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) -sleep 120000 -sql select * from cpustrm -if $rows <= 0 then - return -1 -endi - -sql select * from memstrm -if $rows <= 0 then - return -1 -endi - -sql select * from diskstrm -if $rows <= 0 then - return -1 -endi - -sql select * from bandstrm -if $rows <= 0 then - return -1 -endi - -sql select * from reqstrm -if $rows <= 0 then - return -1 -endi - -sql select * from iostrm -if $rows <= 0 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index cea6d98679945b1d0e8d4dc82f3dd389b3d04a30..a2cf305fae890bf6685962d1c9817bf2a17c54c6 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -105,10 +105,3 @@ sleep 500 run general/parser/sliding.sim sleep 500 run general/parser/function.sim - -#sleep 500 -#run general/parser/repeatStream.sim -#sleep 500 -#run general/parser/stream_on_sys.sim -#sleep 500 -#run general/parser/stream.sim \ No newline at end of file diff --git a/tests/script/general/stream/metrics_n.sim b/tests/script/general/stream/metrics_n.sim deleted file mode 100644 index 7fc08064b29cc6f1681116be16ca460da0e41580..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/metrics_n.sim +++ /dev/null @@ -1,262 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = mn_db -$tbPrefix = mn_tb -$mtPrefix = mn_mt -$stPrefix = mn_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c3 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) -print select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) ===> $data00 $data01 $data02, $data03 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(*), count(tbcol), count(tbcol2) from $mt interval(1d) - -print =============== step3 count32 -#total 32 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -# total 32 count in stream -$st = $stPrefix . c32 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -print =============== step4 count31 - -#total 31 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) -print ===> $data00 $data01 $data02, $data03 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -# total 31 count in stream will crash, 32 will error -$st = $stPrefix . c31 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $mt interval(1d) - -print =============== step5 avg ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . avg -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) - -print =============== step6 others -sql select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $mt interval(1d) -x step6 - return -1 -step6: - -$st = $stPrefix . ot -sql create table $st as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $mt interval(1d) -x step61 - return -1 -step61: - -print =============== step7 where -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 200 then - return -1 -endi - -$st = $stPrefix . wh -#sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) - -print =============== step8 as -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), count(tbcol) from $mt where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 200 then - return -1 -endi - -$st = $stPrefix . as -#sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7, avg(tbcol) as a8, sum(tbcol) as a9, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, count(tbcol) as a7 from $mt where ts < now + 4m interval(1d) - -print =============== step9 -print sleep 120 seconds -sleep 120000 - -print =============== step10 -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi - -$st = $stPrefix . c31 -sql select * from $st -if $data01 != 200 then - return -1 -endi -if $data02 != 200 then - return -1 -endi -if $data03 != 200 then - return -1 -endi -if $data04 != 200 then - return -1 -endi -if $data05 != 200 then - return -1 -endi -if $data06 != 200 then - return -1 -endi -if $data07 != 200 then - return -1 -endi -if $data08 != 200 then - return -1 -endi -if $data09 != 200 then - return -1 -endi - - - -$st = $stPrefix . avg -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 1900 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - diff --git a/tests/script/general/stream/table_1.sim b/tests/script/general/stream/table_1.sim deleted file mode 100644 index f028b1626d156a65a3ff805f1479e9cb178a8d30..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/table_1.sim +++ /dev/null @@ -1,317 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = t1_db -$tbPrefix = t1_tb -$mtPrefix = t1_mt -$stPrefix = t1_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c1 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*) from $tb interval(1d) -print select count(*) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c1 -sql create table $st as select count(*) from $tb interval(1d) - -print =============== step3 c2 -sql select count(tbcol) from $tb interval(1d) -print select count(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c2 -sql create table $st as select count(tbcol) from $tb interval(1d) - -print =============== step4 c3 -sql select count(tbcol2) from $tb interval(1d) -print select count(tbcol2) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(tbcol2) from $tb interval(1d) - -print =============== step5 avg -sql select avg(tbcol) from $tb interval(1d) -print select avg(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 9.500000000 then - return -1 -endi - -$st = $stPrefix . av -sql create table $st as select avg(tbcol) from $tb interval(1d) - -print =============== step6 su -sql select sum(tbcol) from $tb interval(1d) -print select sum(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 190 then - return -1 -endi - -$st = $stPrefix . su -sql create table $st as select sum(tbcol) from $tb interval(1d) - -print =============== step7 mi -sql select min(tbcol) from $tb interval(1d) -print select min(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . mi -sql create table $st as select min(tbcol) from $tb interval(1d) - -print =============== step8 ma -sql select max(tbcol) from $tb interval(1d) -print select max(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . ma -sql create table $st as select max(tbcol) from $tb interval(1d) - -print =============== step9 fi -sql select first(tbcol) from $tb interval(1d) -print select first(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . fi -sql create table $st as select first(tbcol) from $tb interval(1d) - -print =============== step10 la -sql select last(tbcol) from $tb interval(1d) -print select last(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . la -sql create table $st as select last(tbcol) from $tb interval(1d) - -print =============== step11 st -sql select stddev(tbcol) from $tb interval(1d) -print select stddev(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 5.766281297 then - return -1 -endi - -$st = $stPrefix . std -sql create table $st as select stddev(tbcol) from $tb interval(1d) - -print =============== step12 le -sql select leastsquares(tbcol, 1, 1) from $tb interval(1d) -print select leastsquares(tbcol, 1, 1) from $tb interval(1d) ===> $data00 $data01 -#if $data01 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi - -$st = $stPrefix . le -sql create table $st as select leastsquares(tbcol, 1, 1) from $tb interval(1d) - -print =============== step13 -sql select top(tbcol, 1) from $tb interval(1d) - -print =============== step14 -sql select bottom(tbcol, 1) from $tb interval(1d) - -print =============== step15 pe - -sql select percentile(tbcol, 1) from $tb interval(1d) -print select percentile(tbcol, 1) from $tb interval(1d) ===> $data00 $data01 -if $data01 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . pe -sql create table $st as select percentile(tbcol, 1) from $tb interval(1d) - -print =============== step16 -sql select diff(tbcol) from $tb interval(1d) -x step16 - return -1 -step16: - -print =============== step17 wh -sql select count(tbcol) from $tb where ts < now + 4m interval(1d) -print select count(tbcol) from $tb where ts < now + 4m interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . wh -#sql create table $st as select count(tbcol) from $tb where ts < now + 4m interval(1d) - -print =============== step18 as -sql select count(tbcol) from $tb interval(1d) -print select count(tbcol) from $tb interval(1d) ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . as -sql create table $st as select count(tbcol) as c from $tb interval(1d) - -print =============== step19 gb -sql select count(tbcol) from $tb interval(1d) group by tgcol -x step19 - return -1 -step19: - -print =============== step20 x -sql select count(tbcol) from $tb where ts < now + 4m interval(1d) group by tgcol -x step20 - return -1 -step20: - -print =============== step21 -print sleep 120 seconds -sleep 120000 - -print =============== step22 -$st = $stPrefix . c1 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c2 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi - -$st = $stPrefix . av -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 9.500000000 then - return -1 -endi - -$st = $stPrefix . su -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 190 then - return -1 -endi - -$st = $stPrefix . mi -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . ma -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . fi -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0 then - return -1 -endi - -$st = $stPrefix . la -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 19 then - return -1 -endi - -$st = $stPrefix . std -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 5.766281297 then - return -1 -endi - -$st = $stPrefix . le -sql select * from $st -#print ===> select * from $st ===> $data00 $data01 -#if $data01 != @(0.000017, -25270086.331047)@ then -# return -1 -#endi - -$st = $stPrefix . pe -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != 0.190000000 then - return -1 -endi - -#$st = $stPrefix . wh -#sql select * from $st -#print ===> select * from $st ===> $data00 $data01 -#if $data01 != $rowNum then -# return -1 -#endi - -$st = $stPrefix . as -sql select * from $st -print ===> select * from $st ===> $data00 $data01 -if $data01 != $rowNum then - return -1 -endi diff --git a/tests/script/general/stream/table_n.sim b/tests/script/general/stream/table_n.sim deleted file mode 100644 index a336772a9819ec54607265913eb65c26a2ec1a28..0000000000000000000000000000000000000000 --- a/tests/script/general/stream/table_n.sim +++ /dev/null @@ -1,293 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/exec.sh -n dnode1 -s start - -sleep 3000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = tn_db -$tbPrefix = tn_tb -$mtPrefix = tn_mt -$stPrefix = tn_st -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i -$st = $stPrefix . $i - -sql drop databae $db -x step1 -step1: -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int, tbcol2 float) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = -1440 - $y = 0 - while $y < $rowNum - $ms = $x . m - sql insert into $tb values (now $ms , $y , $y ) - $x = $x + 1 - $y = $y + 1 - endw - - $i = $i + 1 -endw - -sleep 100 - -print =============== step2 c3 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) -print select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) ===> $data00 $data01 $data02, $data03 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . c3 -sql create table $st as select count(*), count(tbcol), count(tbcol2) from $tb interval(1d) - -print =============== step3 count32 -#total 32 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -# total 32 count in stream -$st = $stPrefix . c32 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -print =============== step4 count31 - -#total 31 count in select -sql select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) -print ===> $data00 $data01 $data02, $data03 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -# total 31 count in stream will crash, 32 will error -$st = $stPrefix . c31 -sql create table $st as select count(*), count(tbcol) as c1, count(tbcol2) as c2, count(tbcol) as c3, count(tbcol) as c4, count(tbcol) as c5, count(tbcol) as c6, count(tbcol) as c7, count(tbcol) as c8, count(tbcol) as c9, count(tbcol) as c10, count(tbcol) as c11, count(tbcol) as c12, count(tbcol) as c13, count(tbcol) as c14, count(tbcol) as c15, count(tbcol) as c16, count(tbcol) as c17, count(tbcol) as c18, count(tbcol) as c19, count(tbcol) as c20, count(tbcol) as c21, count(tbcol) as c22, count(tbcol) as c23, count(tbcol) as c24, count(tbcol) as c25, count(tbcol) as c26, count(tbcol) as c27, count(tbcol) as c28, count(tbcol) as c29, count(tbcol) as c30 from $tb interval(1d) - -print =============== step5 avg ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $tb interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . avg -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $tb interval(1d) - -print =============== step6 others -sql select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $tb interval(1d) -print ===> $data01 $data02 $data03 -if $data01 != 5.766281297 then - return -1 -endi -#if $data02 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi -if $data03 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . ot -sql create table $st as select stddev(tbcol), leastsquares(tbcol, 1, 1), percentile(tbcol, 1) from $tb interval(1d) - -print =============== step7 total ... -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 5.766281297 then - return -1 -endi -if $data08 != 0.190000000 then - return -1 -endi -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . to -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb interval(1d) - -print =============== step8 where -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . wh -sql create table $st as select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) - - -print =============== step9 as -sql select avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol), stddev(tbcol), percentile(tbcol, 1), count(tbcol), leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m interval(1d) -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data09 != 20 then - return -1 -endi - -$st = $stPrefix . as -#sql create table $st as select avg(tbcol) as a1, sum(tbcol) as a2, min(tbcol) as a3, max(tbcol) as a4, first(tbcol) as a5, last(tbcol) as a6, stddev(tbcol) as a7, percentile(tbcol, 1) as a8, count(tbcol) as a9, leastsquares(tbcol, 1, 1) as a10 from $tb where ts < now + 4m interval(1d) - -print =============== step10 -print sleep 120 seconds -sleep 120000 - -print =============== step11 -$st = $stPrefix . c3 -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . c31 -sql select * from $st -if $data01 != $rowNum then - return -1 -endi -if $data02 != $rowNum then - return -1 -endi -if $data03 != $rowNum then - return -1 -endi - -$st = $stPrefix . avg -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi - -$st = $stPrefix . ot -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 5.766281297 then - return -1 -endi -#if $data02 != @(0.000017, -25362055.126740)@ then -# return -1 -#endi -if $data03 != 0.190000000 then - return -1 -endi - -$st = $stPrefix . to -sql select * from $st -print ===> select * from $st -print ===> $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 -if $data01 != 9.500000000 then - return -1 -endi -if $data02 != 190 then - return -1 -endi -if $data03 != 0 then - return -1 -endi -if $data04 != 19 then - return -1 -endi -if $data05 != 0 then - return -1 -endi -if $data06 != 19 then - return -1 -endi -if $data07 != 5.766281297 then - return -1 -endi -if $data08 != 0.190000000 then - return -1 -endi -if $data09 != 20 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stream/testSuite.sim b/tests/script/general/stream/testSuite.sim index 44f886d02b2831bbd30e7d52a381cd6ea9b7ba27..4a9912b848f1976903199ad314352e70db0ca059 100644 --- a/tests/script/general/stream/testSuite.sim +++ b/tests/script/general/stream/testSuite.sim @@ -1,8 +1,5 @@ run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/general/table/delete_writing.sim b/tests/script/general/table/delete_writing.sim index b55d55eb8a6ef77f453f0d60152d6e1fb7a3269c..2a7fb09104391c0bf7dd04ed94ca6e710c3d9bb2 100644 --- a/tests/script/general/table/delete_writing.sim +++ b/tests/script/general/table/delete_writing.sim @@ -21,8 +21,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start - -sleep 3000 sql connect sql create database db diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim index c6f7402b874115811e6422894c612c341ea5256f..3a8952391818bc02244f7455f694b6c1fe834288 100644 --- a/tests/script/general/wal/sync.sim +++ b/tests/script/general/wal/sync.sim @@ -31,7 +31,6 @@ system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032 print ============== deploy system sh/exec.sh -n dnode1 -s start -sleep 5001 sql connect sql create dnode $hostname2 @@ -43,8 +42,8 @@ print =============== step1 $x = 0 show1: $x = $x + 1 - sleep 2000 - if $x == 5 then + sleep 1000 + if $x == 10 then return -1 endi sql show mnodes -x show1 @@ -82,7 +81,7 @@ restful d1 table_rest 1591772800 30000 restful d1 table_rest 1591872800 30000 restful d1 table_rest 1591972800 30000 -sleep 1000 +sleep 100 sql select * from table_rest; print rows: $rows if $rows != 300000 then @@ -91,29 +90,51 @@ endi print =============== step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode1 -s start -x SIGINT -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi print =============== step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then return -1 endi system sh/exec.sh -n dnode2 -s start -x SIGINT -sleep 5000 +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a2 +endi print =============== step5 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql select * from table_rest; print rows: $rows if $rows != 300000 then diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 34b057e71b19ec36643c73c4fc5e0913dc6ed0a0..efe437fc1d319796dbf6179d570a38107618bc69 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -286,8 +286,12 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim ./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/lossdata.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim ./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim @@ -315,6 +319,7 @@ cd ../../../debug; make ./test.sh -f unique/mnode/mgmt24.sim ./test.sh -f unique/mnode/mgmt25.sim ./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt30.sim ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim @@ -326,16 +331,12 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -./test.sh -f general/parser/stream_on_sys.sim ./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim ./test.sh -f general/stream/metrics_replica1_vnoden.sim ./test.sh -f general/stream/restart_stream.sim ./test.sh -f general/stream/stream_3.sim ./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim ./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim ./test.sh -f general/stream/table_replica1_vnoden.sim ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt index 765e7139163952c43569e596d8ca8cf27d583cf6..9820a00f40b60263537f98cafb098b34b57032ff 100644 --- a/tests/script/jenkins/basic_1.txt +++ b/tests/script/jenkins/basic_1.txt @@ -150,6 +150,8 @@ ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim +./test.sh -f general/db/nosuchfile.sim +./test.sh -f general/parser/function.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim diff --git a/tests/script/jenkins/basic_2.txt b/tests/script/jenkins/basic_2.txt index 014313fafe9c7974baa7205cca41492dedee213a..4f5651219bf3fc8d87e9f95c73963ed16a2f28fd 100644 --- a/tests/script/jenkins/basic_2.txt +++ b/tests/script/jenkins/basic_2.txt @@ -26,6 +26,9 @@ cd ../../../debug; make ./test.sh -f general/tag/set.sim ./test.sh -f general/tag/smallint.sim ./test.sh -f general/tag/tinyint.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim ./test.sh -f general/user/authority.sim ./test.sh -f general/user/monitor.sim @@ -87,4 +90,15 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica2_repeat.sim ./test.sh -f unique/vnode/replica3_basic.sim ./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim \ No newline at end of file +./test.sh -f unique/vnode/replica3_vgroup.sim + +./test.sh -f unique/dnode/monitor.sim +./test.sh -f unique/dnode/monitor_bug.sim +./test.sh -f unique/dnode/simple.sim +./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt index 23d1e0a17df618d3442dbaa62247785be8d6fda7..b44a2c6d44611da0360fe0a2c4266fd92f99aee2 100644 --- a/tests/script/jenkins/basic_3.txt +++ b/tests/script/jenkins/basic_3.txt @@ -1,38 +1,4 @@ -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -#./test.sh -f unique/mnode/mgmt25.sim -#./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - - - -./test.sh -f general/parser/stream_on_sys.sim -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -70,7 +36,7 @@ ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim -#./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim +./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim ./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim @@ -79,3 +45,31 @@ ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +./test.sh -f unique/stable/balance_replica1.sim +./test.sh -f unique/stable/dnode2_stop.sim +./test.sh -f unique/stable/dnode2.sim +./test.sh -f unique/stable/dnode3.sim +./test.sh -f unique/stable/replica2_dnode4.sim +./test.sh -f unique/stable/replica2_vnode3.sim +./test.sh -f unique/stable/replica3_dnode6.sim +./test.sh -f unique/stable/replica3_vnode3.sim + +./test.sh -f unique/mnode/mgmt20.sim +./test.sh -f unique/mnode/mgmt21.sim +./test.sh -f unique/mnode/mgmt22.sim +./test.sh -f unique/mnode/mgmt23.sim +./test.sh -f unique/mnode/mgmt24.sim +./test.sh -f unique/mnode/mgmt25.sim +./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt33.sim +./test.sh -f unique/mnode/mgmt34.sim +./test.sh -f unique/mnode/mgmtr2.sim + +./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim +./test.sh -f general/stream/restart_stream.sim +./test.sh -f general/stream/stream_3.sim +./test.sh -f general/stream/stream_restart.sim +./test.sh -f general/stream/table_del.sim +./test.sh -f general/stream/table_replica1_vnoden.sim diff --git a/tests/script/jenkins/simple.txt b/tests/script/jenkins/simple.txt index 2fe364bf26e0b3edf83278b085503e6ad2ac38f0..88ff4d660131bea5dee2f5ccd0dad9d81c7bed5f 100644 --- a/tests/script/jenkins/simple.txt +++ b/tests/script/jenkins/simple.txt @@ -147,9 +147,7 @@ cd ../../../debug; make ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim ./test.sh -f general/parser/bug.sim -#./test.sh -f general/parser/stream_on_sys.sim ./test.sh -f general/parser/repeatAlter.sim -#./test.sh -f general/parser/repeatStream.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index b271f5b726bd6e9e50265420ec6cfa6695b0f551..372bdc9d9d55635de6f36a0629b2f9c2aca5df07 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -10,6 +10,7 @@ cd ../../../debug; make ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim ./test.sh -f unique/cluster/cache.sim +./test.sh -f unique/cluster/vgroup100.sim ./test.sh -f unique/column/replica3.sim @@ -25,12 +26,21 @@ cd ../../../debug; make ./test.sh -f unique/db/replica_part.sim ./test.sh -f unique/dnode/alternativeRole.sim +./test.sh -f unique/dnode/monitor.sim +./test.sh -f unique/dnode/monitor_bug.sim +./test.sh -f unique/dnode/simple.sim ./test.sh -f unique/dnode/balance1.sim ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/lossdata.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f unique/dnode/reason.sim ./test.sh -f unique/dnode/remove1.sim ./test.sh -f unique/dnode/remove2.sim ./test.sh -f unique/dnode/vnode_clean.sim @@ -50,12 +60,14 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt20.sim ./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim ./test.sh -f unique/mnode/mgmt25.sim ./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt30.sim ./test.sh -f unique/mnode/mgmt33.sim ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim diff --git a/tests/script/regressionSuite.sim b/tests/script/regressionSuite.sim index ff1f9f53559c6da89ed2cdaf77c29f1859578ae6..e5e2194e87537f8496c9f6c5832d1a7d71842494 100644 --- a/tests/script/regressionSuite.sim +++ b/tests/script/regressionSuite.sim @@ -134,9 +134,6 @@ run general/parser/tags_dynamically_specifiy.sim run general/parser/set_tag_vals.sim run general/parser/repeatAlter.sim ##unsupport run general/parser/slimit_alter_tags.sim -##unsupport run general/parser/stream_on_sys.sim -run general/parser/stream.sim -#unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim run general/stable/metrics.sim @@ -214,14 +211,8 @@ run general/vector/table_mix.sim run general/vector/table_query.sim run general/vector/table_time.sim run general/stream/restart_stream.sim -run general/stream/stream_1.sim -run general/stream/stream_2.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim -run general/stream/table_1.sim -run general/stream/metrics_1.sim -run general/stream/table_n.sim -run general/stream/metrics_n.sim run general/stream/table_del.sim run general/stream/metrics_del.sim run general/stream/table_replica1_vnoden.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index cd2f3772eb11a0c5d3a7a69fcb3d62162eadf14c..9b61a33d45f69a200b3f364f1553bdec5a3f52b5 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -132,10 +132,12 @@ echo "cqdebugFlag 143" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG +echo "slaveQuery 0" >> $TAOS_CFG echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG echo "defaultPass taosdata" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG +echo "balanceInterval 1" >> $TAOS_CFG echo "clog 2" >> $TAOS_CFG #echo "cache 1" >> $TAOS_CFG echo "days 10" >> $TAOS_CFG diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index c647a35a5cc21c3a071b67b27330418a7b7ac57a..c9c85cb45dec7dd822b93916c452c274f2361ec9 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -360,12 +360,14 @@ print ============== step7: stop dnode3/dnode2, and cluster unable to provide se system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT sleep 3000 -sql_error select count(*) from $stb +sql select count(*) from $stb -x s71 +s71: print ============== step8: restart dnode2, and cluster Still unable to provide services system sh/exec.sh -n dnode2 -s start sleep 3000 -sql_error select count(*) from $stb +sql select count(*) from $stb -x s81 +s81: print ============== step9: restart dnode3, and cluster Resume service delivery system sh/exec.sh -n dnode3 -s start diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 23ace47e3cf13cb18467c0482a30be97d5f03d33..387ae4625f392b8f108d7e9c34645905348d5e45 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -45,15 +45,30 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ============== step2: start dnode2 and add into cluster , then create database with replica 1, and create table, insert data system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 -$sleepTimer = 10000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi $db = db sql create database $db replica 1 @@ -84,7 +99,6 @@ while $i < $tblNum endw sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 @@ -92,118 +106,65 @@ endi print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep $sleepTimer - - -$loopCnt = 0 -wait_dnode2_offline_0: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_offline_0 -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline_0 -endi - system sh/exec.sh -n dnode2 -s start -sleep $sleepTimer +$x = 0 +a0: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -$loopCnt = 0 -wait_dnode2_reready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi -sql show dnodes -if $rows != 2 then - sleep 2000 - goto wait_dnode2_reready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -#$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_reready +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a0 endi sql select count(*) from $stb -sleep 1000 print data00 $data00 if $data00 != $totalRows then return -1 endi - print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 3000 -sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode3_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode3_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode3_ready +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_3 != ready then + goto step2 endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode3_ready + +sql alter database $db replica 2 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -211,44 +172,22 @@ if $data00 != $totalRows then return -1 endi - print ============== step4: stop dnode2 for checking if sync ok system sh/exec.sh -n dnode2 -s stop -sleep $sleepTimer -$loopCnt = 0 -wait_dnode2_offline: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_offline -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != offline then - sleep 2000 - goto wait_dnode2_offline -endi -if $dnode3Status != ready then - sleep 2000 - goto wait_dnode2_offline +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a2 endi -sleep $sleepTimer # waitting for move master vnode of dnode2 to dnode3 # check using select sql select count(*) from $stb print data00 $data00 @@ -258,38 +197,20 @@ endi print ============== step5: restart dnode2 system sh/exec.sh -n dnode2 -s start -sleep 3000 - -$loopCnt = 0 -wait_dnode2_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show dnodes -if $rows != 3 then - sleep 2000 - goto wait_dnode2_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -#print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -#$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode2Status != ready then - sleep 2000 - goto wait_dnode2_ready +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -300,40 +221,39 @@ endi print ============== step6: start dnode4 and add into cluster , then alter replica from 2 to 3, and waiting sync system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 - -sql alter database $db replica 3 -sleep $sleepTimer -$loopCnt = 0 -wait_dnode4_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes -if $rows != 4 then - sleep 2000 - goto wait_dnode4_ready +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_4 != ready then + goto step6 endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -#$dnode1Status = $data4_1 -#$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode4Status != ready then - sleep 2000 - goto wait_dnode4_ready + +sql alter database $db replica 3 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 endi -sleep $sleepTimer # check using select sql select count(*) from $stb print data00 $data00 @@ -343,44 +263,20 @@ endi print ============== step7: alter replica from 3 to 2, and waiting sync sql alter database $db replica 2 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_2: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 $data10_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 $data10_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 $data10_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 $data10_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$thirdDnode_2 = $data8_1 -$thirdDnode_3 = $data8_2 -$thirdDnode_4 = $data8_3 -$thirdDnode_5 = $data8_4 - -if $thirdDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 -endi -if $thirdDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_2 +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a5 endi -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -390,85 +286,20 @@ endi print ============== step8: alter replica from 2 to 1, and waiting sync sql alter database $db replica 1 -sleep $sleepTimer -$loopCnt = 0 -wait_vgroups_replic_to_1: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +a6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi -sql show vgroups -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$sencodDnode_2 = $data6_1 -$sencodDnode_3 = $data6_2 -$sencodDnode_4 = $data6_3 -$sencodDnode_5 = $data6_4 - -if $sencodDnode_2 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_3 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_4 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi -if $sencodDnode_5 != null then - sleep 2000 - goto wait_vgroups_replic_to_1 -endi - -$loopCnt = 0 -all_dnodes_ready: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a6 endi -sql show dnodes -if $rows != 4 then - sleep 2000 - goto all_dnodes_ready -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 -#$dnode5Status = $data4_5 - -if $dnode1Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode2Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode3Status != ready then - sleep 2000 - goto all_dnodes_ready -endi -if $dnode4Status != ready then - sleep 2000 - goto all_dnodes_ready -endi - -sleep $sleepTimer #waiting del one replica data # check using select sql select count(*) from $stb print data00 $data00 @@ -481,39 +312,31 @@ sql drop dnode $hostname2 sql drop dnode $hostname3 sleep $sleepTimer -$loopCnt = 0 -wait_dnode23_dropped: -$loopCnt = $loopCnt + 1 -if $loopCnt == 10 then - return -1 -endi +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi sql show dnodes if $rows != 2 then - sleep 2000 - goto wait_dnode23_dropped -endi -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 -print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 -#print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 -#print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$dnode1Status = $data4_1 -$dnode2Status = $data4_2 -$dnode3Status = $data4_3 -$dnode4Status = $data4_4 - -if $dnode2Status != null then - sleep 2000 - goto wait_dnode23_dropped -endi -if $dnode3Status != null then - sleep 2000 - goto wait_dnode23_dropped + goto step9 endi -if $dnode4Status != ready then - return -1 + +$x = 0 +a7: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a7 endi sleep $sleepTimer #waiting move vnode from dnode3/dnode3 to dnode4 @@ -523,3 +346,13 @@ print data00 $data00 if $data00 != $totalRows then return -1 endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index c36e9f015e16512293b3113f9e46de0fd581c4ef..dd868e9eedbfc7bf81ab9667897221a9115015b1 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -140,3 +140,11 @@ if $data00 != $totalRows then return -1 endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index 53209b0d5a07781ff8e22f406fc31823bdacc9b6..b3bd853ffc1f0953f4b021c051dfbbe70783aa6c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -158,13 +158,15 @@ if $dnode4Vtatus != offline then sleep 2000 goto wait_dnode4_vgroup_offline endi -if $dnode3Vtatus != unsynced then +if $dnode3Vtatus != master then sleep 2000 goto wait_dnode4_vgroup_offline endi -sql_error select count(*) from $stb -sql_error insert into $tb values (now, 9988) +sql select count(*) from $stb -x s31 +s31: +#sql_error insert into $tb values (now, 9988) -x s32 +#s32: print ============== step4: restart dnode2, then create database with replica 2, and create table, insert data system sh/exec.sh -n dnode2 -s start diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index ba973cbe06d92d6bd6a007dc5bd3cdb8fa6f2bfd..d4eb360efb2e1bd5687ed6e774736492c482bd3c 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -140,11 +140,9 @@ if $rows != 1 then goto wait_dnode4_vgroup_offline endi print show vgroups: -print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 -print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 -print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data5_2 -$dnode3Vtatus = $data7_2 +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +$dnode4Vtatus = $data05 +$dnode3Vtatus = $data07 if $dnode4Vtatus != offline then sleep 2000 @@ -198,7 +196,3 @@ if $data00 != $totalRows then sleep 2000 goto wait_table_altered endi - - - - diff --git a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim index 3867aa3699cb84881c8b99eaa6ffa0813e8c785f..46d5e34fcbf8d70b4f678fc8e5abf311c9363ce8 100644 --- a/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim +++ b/tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim @@ -14,6 +14,9 @@ sql alter table $stb add column f1 float $tblNum = $totalTableNum $alterTblNum = 10 +sql reset query cache +sleep 100 + $i = 1 while $i < $alterTblNum $tb = tb . $i diff --git a/tests/script/unique/big/balance.sim b/tests/script/unique/big/balance.sim index 91a7c538d27a30433aeea1fd1214e3c2d79283ec..f20939d458d512472cb822b44b5af0b378347017 100644 --- a/tests/script/unique/big/balance.sim +++ b/tests/script/unique/big/balance.sim @@ -21,7 +21,6 @@ system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 1000 print =============== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect $i = 0 @@ -92,8 +91,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show1: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -108,7 +107,7 @@ if $data2_2 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -143,7 +142,6 @@ endi print ========== step2 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 10000 print ========== step3 sql drop dnode $hostname2 @@ -151,8 +149,8 @@ sql drop dnode $hostname2 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -171,10 +169,9 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -212,8 +209,8 @@ sql drop dnode $hostname3 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -228,10 +225,9 @@ if $data2_3 != null then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -267,14 +263,31 @@ print ========== step5 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_4 != ready then + goto step5 +endi + sql alter database db replica 2 $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -289,7 +302,7 @@ if $data2_4 != 4 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum @@ -321,16 +334,14 @@ if $data00 != $totalNum then goto show5 endi - print ========== step6 -sleep 3000 sql alter database db replica 1 $x = 0 show6: $x = $x + 1 - sleep 3000 - if $x == 10 then + sleep 1000 + if $x == 30 then return -1 endi @@ -345,7 +356,7 @@ if $data2_4 != 2 then endi sql reset query cache -sleep 1000 +sleep 100 sql select count(*) from t10 print select count(*) from t10 $data00 expect $rowNum diff --git a/tests/script/unique/big/tcp.sim b/tests/script/unique/big/tcp.sim index 38cb667ec03b400e29cc1017f616434b650375d4..3c5cf92c7f183a84b2f1be0ce389bf86f482a42a 100644 --- a/tests/script/unique/big/tcp.sim +++ b/tests/script/unique/big/tcp.sim @@ -15,7 +15,6 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c debugFlag -v 131 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ======================== dnode1 start @@ -52,7 +51,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step3 $i sql select count(*) from $mt print ===> $data00 $data01 @@ -83,7 +82,7 @@ while $i < $tbNum endw $i = 0 -while $i < 10 +while $i < 5 print =============== step5 $i sql select count(*) from $mt where tgcol < 20200 print ===> $data00 $data01 diff --git a/tests/script/unique/cluster/balance1.sim b/tests/script/unique/cluster/balance1.sim index ec0c58d41651542e03d9f2ee710a38c5744c1f82..c98687a81ceae03d20fc205a121fcfc813c74be7 100644 --- a/tests/script/unique/cluster/balance1.sim +++ b/tests/script/unique/cluster/balance1.sim @@ -84,16 +84,14 @@ endi print ============================== step2 print ========= start dnode2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 5000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -129,13 +127,12 @@ sql insert into c_b1_t3 values(1520000024031, 31) print ============================== step4 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -164,16 +161,14 @@ print dnode4 ==> $dnode4Role print ============================== step5 print ========= add dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -212,13 +207,12 @@ sql_error create dnode $hostname1 print ============================== step8 sql drop dnode $hostname3 -sleep 15000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -253,13 +247,12 @@ endi print ============================== step9 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show9 @@ -302,8 +295,8 @@ sql insert into c_b1_t4 values(1520000024041, 41) $x = 0 show10: $x = $x + 1 - sleep 2000 - if $x == 30 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show10 @@ -335,7 +328,23 @@ sql_error create table c_b1_t5 (t timestamp, i int) -x error3 print ============================== step13 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 +$x = 0 +step13: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode5 $data4_5 + +if $data4_5 != ready then + goto step13 +endi sql show mnodes $dnode1Role = $data2_1 @@ -371,19 +380,35 @@ $dnode5Vnodes = $data2_5 print dnode5 $dnode2Vnodes if $dnode1Vnodes != 2 then - return -1 + goto step13 endi if $dnode4Vnodes != 2 then - return -1 + goto step13 endi if $dnode5Vnodes != 2 then - return -1 + goto step13 endi print ============================== step14 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 15000 +$x = 0 +step14: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 +print dnode4 $data4_5 + +if $data4_6 != ready then + goto step14 +endi sql create database c_b1_d7 sql use c_b1_d7 @@ -406,7 +431,7 @@ sql insert into c_b1_t8 values(1520000024081, 81) $x = 0 show14: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -440,7 +465,7 @@ print ============================== step17 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql use c_b1_d1 sql select * from c_b1_d1.c_b1_t1 diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index b8477965c6a1bb8a5fca0982ef25bbf993cf6ea2..ffd13445edbf76dbc96b82a14848ef6b788237fa 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -50,12 +50,48 @@ print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 4001 sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 4001 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi + sql create database c_b2_d1 replica 2 sql use c_b2_d1 @@ -112,13 +148,12 @@ endi print ============================== step2 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show2 @@ -155,13 +190,12 @@ print ============================== step3 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 10000 $x = 0 show3: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi sql show dnodes -x show3 @@ -209,13 +243,12 @@ endi print ============================== step4 print ========= drop dnode3 sql drop dnode $hostname3 -sleep 9000 $x = 0 show4: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show4 @@ -266,12 +299,11 @@ print ============================== step5 print ========= start dnode3 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 30 then return -1 endi @@ -304,7 +336,7 @@ print dnode5 ==> $dnode5Role print ============================== step6 system sh/exec.sh -n dnode1 -s stop -x SIGINT print stop dnode1 and sleep 10000 -sleep 10000 +sleep 5000 sql drop dnode $hostname1 print drop dnode1 and sleep 9000 @@ -364,7 +396,7 @@ print ============================== step12 print ========= check data sql reset query cache -sleep 1000 +sleep 100 sql select * from c_b2_d1.c_b2_t1 order by t desc print $data01 $data11 $data21 $data31 $data41 diff --git a/tests/script/unique/cluster/balance3.sim b/tests/script/unique/cluster/balance3.sim index e3b8125d8c34a7e59aaac4f3df56e4650fb6cc68..c2e9a845149cd9df78b9528115c5108c1e6531d3 100644 --- a/tests/script/unique/cluster/balance3.sim +++ b/tests/script/unique/cluster/balance3.sim @@ -7,7 +7,6 @@ system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 system sh/deploy.sh -n dnode6 -i 6 system sh/deploy.sh -n dnode7 -i 7 -system sh/deploy.sh -n dnode8 -i 8 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 @@ -16,7 +15,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode6 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode7 -c numOfMnodes -v 3 -system sh/cfg.sh -n dnode8 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0 @@ -25,7 +23,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 0 -system sh/cfg.sh -n dnode8 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode1 -c wallevel -v 1 system sh/cfg.sh -n dnode2 -c wallevel -v 1 @@ -34,19 +31,52 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 1 system sh/cfg.sh -n dnode5 -c wallevel -v 1 system sh/cfg.sh -n dnode6 -c wallevel -v 1 system sh/cfg.sh -n dnode7 -c wallevel -v 1 -system sh/cfg.sh -n dnode8 -c wallevel -v 1 print ============== step1 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 2001 - sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3001 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi sql create database c_b3_d1 replica 3 sql use c_b3_d1 @@ -89,8 +119,6 @@ $dnode2Vnodes = $data2_2 print dnode2 $dnode2Vnodes $dnode3Vnodes = $data2_3 print dnode3 $dnode3Vnodes -$dnode4Vnodes = $data2_4 -print dnode4 $dnode4Vnodes if $dnode1Vnodes != 3 then goto show1 @@ -101,30 +129,22 @@ endi if $dnode3Vnodes != 3 then goto show1 endi -if $dnode4Vnodes != null then - goto show1 -endi sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step2 print ========= start dnode4 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 9000 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show2 @@ -145,21 +165,16 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 print ============================== step3 print ========= drop dnode2 sql drop dnode $hostname2 -sleep 9000 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show3 @@ -189,25 +204,21 @@ sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 -print dnode4 ==> $data2_4 -print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 system sh/exec.sh -n dnode2 -s stop -x SIGINT print ============================== step4 sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start -sleep 10000 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi + sql show dnodes -x show4 $dnode1Vnodes = $data2_1 print dnode1 $dnode1Vnodes @@ -228,8 +239,10 @@ print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 print dnode4 ==> $data2_4 print dnode5 ==> $data2_5 -print dnode6 ==> $data2_6 -print dnode7 ==> $data2_7 + +if $data2_4 != slave then + goto show4 +endi print ============================== step5 print ========= drop dnode3 @@ -239,8 +252,8 @@ sleep 9000 $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show5 @@ -277,16 +290,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_5 != slave then + goto show5 +endi + print ============================== step6 sql create dnode $hostname6 system sh/exec.sh -n dnode6 -s start -sleep 9000 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show6 @@ -299,6 +315,15 @@ print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 print dnode6 $dnode6Vnodes +if $dnode1Vnodes != 2 then + goto show6 +endi +if $dnode4Vnodes != 2 then + goto show6 +endi +if $dnode5Vnodes != 3 then + goto show6 +endi if $dnode6Vnodes != 2 then goto show6 endi @@ -315,13 +340,12 @@ print dnode7 ==> $data2_7 print ============================== step7 print ========= drop dnode4 sql drop dnode $hostname4 -sleep 9000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show7 @@ -357,16 +381,19 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 +if $data2_6 != slave then + goto show7 +endi + print ============================== step8 sql create dnode $hostname7 system sh/exec.sh -n dnode7 -s start -sleep 9000 $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes -x show8 @@ -379,6 +406,15 @@ print dnode6 $dnode6Vnodes $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes +if $dnode1Vnodes != 2 then + goto show8 +endi +if $dnode5Vnodes != 2 then + goto show8 +endi +if $dnode6Vnodes != 3 then + goto show8 +endi if $dnode7Vnodes != 2 then goto show8 endi @@ -393,19 +429,17 @@ print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 print ============================== step9 -print ========= drop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT -print stop dnode1 and sleep 10000 -sleep 10000 -sql drop dnode $hostname1 -print drop dnode1 and sleep 9000 -sleep 9000 +$x = 0 +show9: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show mnodes -x show9 -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 @@ -414,26 +448,27 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -if $dnode1Role != offline then - return -1 +if $data2_1 != offline then + goto show9 +endi +if $data2_5 != master then + goto show9 +endi +if $data2_6 != slave then + goto show9 endi -print ============================== step9.1 -sleep 2000 -system sh/exec.sh -n dnode1 -s start - +print ============================== step10 +sql drop dnode $hostname1 $x = 0 -show9: +show10: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi +sql show mnodes -x show10 -sql show mnodes -$dnode1Role = $data2_1 -$dnode4Role = $data2_4 -$dnode5Role = $data2_5 print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 @@ -442,7 +477,31 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -sql show dnodes -x show9 +if $data2_1 != null then + goto show10 +endi +if $data2_5 != master then + goto show10 +endi +if $data2_6 != slave then + goto show10 +endi +if $data2_7 != slave then + goto show10 +endi + +print ============================== step11 +system sh/exec.sh -n dnode1 -s start + +$x = 0 +show11: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes -x show11 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -451,17 +510,16 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 3 then - goto show9 + goto show11 endi if $dnode6Vnodes != 3 then - goto show9 + goto show11 endi if $dnode7Vnodes != 3 then - goto show9 + goto show11 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 sql show mnodes print dnode1 ==> $data2_1 @@ -472,15 +530,13 @@ print dnode5 ==> $data2_5 print dnode6 ==> $data2_6 print dnode7 ==> $data2_7 -print ============================== step11 -print ========= add db4 - +print ============================== step12 sql create database c_b3_d4 replica 3 sql use c_b3_d4 $x = 0 create4: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -491,16 +547,14 @@ sql insert into c_b3_t4 values(1520000022043, 43) sql insert into c_b3_t4 values(1520000023042, 42) sql insert into c_b3_t4 values(1520000024041, 41) -sleep 3000 - $x = 0 -show11: +show12: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi -sql show dnodes -x show11 +sql show dnodes -x show12 $dnode5Vnodes = $data2_5 print dnode5 $dnode5Vnodes $dnode6Vnodes = $data2_6 @@ -509,21 +563,18 @@ $dnode7Vnodes = $data2_7 print dnode7 $dnode7Vnodes if $dnode5Vnodes != 4 then - goto show11 + goto show12 endi if $dnode6Vnodes != 4 then - goto show11 + goto show12 endi if $dnode7Vnodes != 4 then - goto show11 + goto show12 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 - print ============================== step13 sql reset query cache -sleep 1000 +sleep 200 print ========= check data @@ -590,4 +641,3 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/cluster/vgroup100.sim b/tests/script/unique/cluster/vgroup100.sim index 68a5bad6b32dc23c667900707d3e9027ac943077..656ed2ec44ebbb2d666ed2618e15d5ae34a5e525 100644 --- a/tests/script/unique/cluster/vgroup100.sim +++ b/tests/script/unique/cluster/vgroup100.sim @@ -17,7 +17,6 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 0 print ============================== step1 system sh/exec.sh -n dnode1 -s start -sleep 2000 sql connect print ============================== step2 @@ -27,8 +26,6 @@ system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 - $maxNum = 102 $maxNum = 12 @@ -92,13 +89,11 @@ print ============================== step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 print ============================== step6 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 10000 print ============================== step7 diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index fee47caadb7ab4cdab9db59b804a03eeb07a0149..f84339be79ad08951ff60c0df0d8992027c2ef98 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -18,7 +18,6 @@ system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 @@ -38,8 +37,6 @@ while $i < 2000 $i = $i + 1 endw -sleep 2500 - sql show db.vgroups if $rows != 2 then return -1 @@ -73,7 +70,6 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start @@ -92,7 +88,6 @@ if $data2_1 != master then goto step3 endi -sleep 1000 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/unique/db/delete_part.sim b/tests/script/unique/db/delete_part.sim index fd92c160d9fcc8f2b5f2e93e2d656c0f48a8e80e..682444e922a5e1403ac6046b85f1c48b0f4bc78a 100644 --- a/tests/script/unique/db/delete_part.sim +++ b/tests/script/unique/db/delete_part.sim @@ -31,11 +31,29 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + $loop = 0 begin: @@ -53,7 +71,6 @@ begin: sql insert into t13 values(now, 1 ) sql create table t14 (ts timestamp, i int) sql insert into t14 values(now, 1 ) - sleep 1200 sql create table t21 (ts timestamp, i int) sql insert into t21 values(now, 1 ) @@ -63,7 +80,6 @@ begin: sql insert into t23 values(now, 1 ) sql create table t24 (ts timestamp, i int) sql insert into t24 values(now, 1 ) - sleep 1200 sql create table t31 (ts timestamp, i int) sql insert into t31 values(now, 1 ) @@ -73,7 +89,6 @@ begin: sql insert into t33 values(now, 1 ) sql create table t34 (ts timestamp, i int) sql insert into t34 values(now, 1 ) - sleep 1200 sql create table t41 (ts timestamp, i int) sql insert into t41 values(now, 1 ) @@ -83,7 +98,6 @@ begin: sql insert into t43 values(now, 1 ) sql create table t44 (ts timestamp, i int) sql insert into t44 values(now, 1 ) - sleep 1200 sql create table t51 (ts timestamp, i int) sql insert into t51 values(now, 1 ) @@ -93,7 +107,6 @@ begin: sql insert into t53 values(now, 1 ) sql create table t54 (ts timestamp, i int) sql insert into t54 values(now, 1 ) - sleep 1200 sql create table t61 (ts timestamp, i int) sql insert into t61 values(now, 1 ) @@ -103,7 +116,6 @@ begin: sql insert into t63 values(now, 1 ) sql create table t64 (ts timestamp, i int) sql insert into t64 values(now, 1 ) - sleep 1200 sql create table t71 (ts timestamp, i int) sql insert into t71 values(now, 1 ) @@ -113,7 +125,6 @@ begin: sql insert into t73 values(now, 1 ) sql create table t74 (ts timestamp, i int) sql insert into t74 values(now, 1 ) - sleep 1200 sql create table t81 (ts timestamp, i int) sql insert into t81 values(now, 1 ) @@ -123,7 +134,6 @@ begin: sql insert into t83 values(now, 1 ) sql create table t84 (ts timestamp, i int) sql insert into t84 values(now, 1 ) - sleep 1200 sql show dnodes print dnode1 openVnodes $data2_1 @@ -138,29 +148,36 @@ begin: print ======== step2 $loop system sh/exec.sh -n dnode2 -s stop - sleep 1000 print ==> drop database $db sql drop database $db print ======== step3 $loop sleep 2000 system sh/exec.sh -n dnode2 -s start - sleep 15000 - sql show dnodes + + $x = 0 + step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + + sql show dnodes -x step3 print dnode1 openVnodes $data2_1 $data4_1 print dnode2 openVnodes $data2_2 $data4_2 if $data2_1 != 0 then - return -1 + goto step3 endi if $data2_2 != 0 then - return -1 + goto step3 endi if $data4_1 != ready then - return -1 + goto step3 endi if $data4_2 != ready then - return -1 + goto step3 endi print ===> test times : $loop @@ -171,7 +188,7 @@ begin: $loop = $loop + 1 sql reset query cache - sleep 1000 + sleep 100 goto begin diff --git a/tests/script/unique/db/replica_add12.sim b/tests/script/unique/db/replica_add12.sim index 62d7e9daa32f726a2d72bd7dbe606652ab602ab5..d46187e4456acbd39c7b1f7689388af156d04f85 100644 --- a/tests/script/unique/db/replica_add12.sim +++ b/tests/script/unique/db/replica_add12.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -27,7 +32,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -65,8 +106,6 @@ if $rows != 1 then return -1 endi -sleep 2000 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -89,10 +128,40 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 10000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ======== step3 - sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -115,7 +184,6 @@ sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -138,33 +206,54 @@ if $rows != 2 then endi sql reset query cache -sleep 2000 +sleep 200 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sql select * from d1.t1 -x s51 +s51: +#sql insert into d1.t1 values(now, 3) -x s52 +s52: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +print ========= step6 +system sh/exec.sh -n dnode2 -s start + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -print ===== insert data +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) -sleep 1000 +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -print ========= step6 -system sh/exec.sh -n dnode2 -s start -sleep 5000 +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi -sql insert into d1.t1 values(now, 3) -sql insert into d2.t2 values(now, 3) -sql insert into d3.t3 values(now, 3) -sql insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(now, 3) -x step6 +sql insert into d2.t2 values(now, 3) -x step6 +sql insert into d3.t3 values(now, 3) -x step6 +sql insert into d4.t4 values(now, 3) -x step6 sql select * from d1.t1 if $rows != 3 then @@ -187,30 +276,52 @@ if $rows != 3 then endi print ========= step61 - system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) -sleep 1000 +#sql insert into d1.t1 values(now, 3) -x s61 +s61: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d2.t2 -x s62 +s62: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 -sql insert into d1.t1 values(now, 5) -sql insert into d2.t2 values(now, 5) -sql insert into d3.t3 values(now, 5) -sql insert into d4.t4 values(now, 5) -sleep 1000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql insert into d1.t1 values(now, 5) -x step7 +sql insert into d2.t2 values(now, 5) -x step7 +sql insert into d3.t3 values(now, 5) -x step7 +sql insert into d4.t4 values(now, 5) -x step7 sql select * from d1.t1 if $rows != 4 then @@ -235,8 +346,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add13.sim b/tests/script/unique/db/replica_add13.sim index 5d34440e3e333935cfbbff00baeeec045c9f9e51..13a5c9783228761c2b431f9bb1f571c29c05f080 100644 --- a/tests/script/unique/db/replica_add13.sim +++ b/tests/script/unique/db/replica_add13.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,41 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 1 @@ -46,7 +85,6 @@ sql insert into d1.t1 values(1589529000011, 1) sql insert into d2.t2 values(1589529000021, 1) sql insert into d3.t3 values(1589529000031, 1) sql insert into d4.t4 values(1589529000041, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -73,17 +111,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 -print ======== step3 $x = 0 -show3: +a1: $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 + sleep 1000 + if $x == 40 then + return -1 endi - + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +print ======== step3 sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -111,7 +172,6 @@ sql insert into d1.t1 values(1589529000012, 2) sql insert into d2.t2 values(1589529000022, 2) sql insert into d3.t3 values(1589529000032, 2) sql insert into d4.t4 values(1589529000042, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -135,15 +195,13 @@ endi print ========= step5 sql reset query cache -sleep 1000 +sleep 100 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000013, 3) sql insert into d2.t2 values(1589529000023, 3) sql insert into d3.t3 values(1589529000033, 3) sql insert into d4.t4 values(1589529000043, 3) -sleep 1000 sql select * from d1.t1 if $rows != 3 then @@ -167,15 +225,45 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000014, 4) sql insert into d2.t2 values(1589529000024, 4) sql insert into d3.t3 values(1589529000034, 4) sql insert into d4.t4 values(1589529000044, 4) -sleep 1000 sql select * from d1.t1 print select * from d1.t1 $rows @@ -203,15 +291,45 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000015, 5) sql insert into d2.t2 values(1589529000025, 5) sql insert into d3.t3 values(1589529000035, 5) sql insert into d4.t4 values(1589529000045, 5) -sleep 1000 sql select * from d1.t1 if $rows != 5 then @@ -235,15 +353,45 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1589529000016, 6) sql insert into d2.t2 values(1589529000026, 6) sql insert into d3.t3 values(1589529000036, 6) sql insert into d4.t4 values(1589529000046, 6) -sleep 1000 sql select * from d1.t1 if $rows != 6 then @@ -268,8 +416,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_add23.sim b/tests/script/unique/db/replica_add23.sim index 9285f681374b7524b87c749f56b4bff92b1fdcca..ac0bd6d9d7b09065c832fb8ad0698b97a9f94853 100644 --- a/tests/script/unique/db/replica_add23.sim +++ b/tests/script/unique/db/replica_add23.sim @@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -29,7 +34,42 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -46,7 +86,6 @@ sql insert into d1.t1 values(1588262400001, 1) sql insert into d2.t2 values(1588262400001, 1) sql insert into d3.t3 values(1588262400001, 1) sql insert into d4.t4 values(1588262400001, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -73,17 +112,40 @@ sql alter database d1 replica 3 sql alter database d2 replica 3 sql alter database d3 replica 3 sql alter database d4 replica 3 -sleep 10000 -print ======== step3 $x = 0 -show3: +a1: $x = $x + 1 - sleep 2000 - if $x == 10 then - return -1 + sleep 1000 + if $x == 40 then + return -1 endi +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a1 +endi + +print ======== step3 sql show dnodes print dnode1 ==> openVnodes: $data2_1 print dnode2 ==> openVnodes: $data2_2 @@ -111,7 +173,6 @@ sql insert into d1.t1 values(1588262400002, 2) sql insert into d2.t2 values(1588262400002, 2) sql insert into d3.t3 values(1588262400002, 2) sql insert into d4.t4 values(1588262400002, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -134,17 +195,14 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 100 print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql insert into d1.t1 values(1588262400003, 3) sql insert into d2.t2 values(1588262400003, 3) sql insert into d3.t3 values(1588262400003, 3) sql insert into d4.t4 values(1588262400003, 3) -sleep 1000 sql select * from d1.t1 if $rows != 3 then @@ -168,15 +226,45 @@ endi print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400004, 4) sql insert into d2.t2 values(1588262400004, 4) sql insert into d3.t3 values(1588262400004, 4) sql insert into d4.t4 values(1588262400004, 4) -sleep 1000 sql select * from d1.t1 if $rows != 4 then @@ -200,15 +288,45 @@ endi print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step7 +endi + system sh/exec.sh -n dnode4 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400005, 5) sql insert into d2.t2 values(1588262400005, 5) sql insert into d3.t3 values(1588262400005, 5) sql insert into d4.t4 values(1588262400005, 5) -sleep 1000 sql select * from d1.t1 if $rows != 5 then @@ -232,15 +350,45 @@ endi print ========= step8 system sh/exec.sh -n dnode4 -s start -sleep 5000 + +$x = 0 +step8: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(1588262400006, 6) sql insert into d2.t2 values(1588262400006, 6) sql insert into d3.t3 values(1588262400006, 6) sql insert into d4.t4 values(1588262400006, 6) -sleep 1000 sql select * from d1.t1 if $rows != 6 then @@ -265,8 +413,4 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_part.sim b/tests/script/unique/db/replica_part.sim index 5c5f98a10845b4c214331c4b8019d820083580c9..9880ec666c8543676c406c843d14235a09ae13ee 100644 --- a/tests/script/unique/db/replica_part.sim +++ b/tests/script/unique/db/replica_part.sim @@ -14,6 +14,10 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +25,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -38,7 +73,6 @@ sql insert into d2.t2 values(now, 1) sql insert into d1.t1 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -61,24 +95,85 @@ if $rows != 1 then endi print ========= step2 alter db -system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========= step3 system sh/exec.sh -n dnode2 -s start -sleep 10000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step3 +endi print ========= step4 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -102,44 +197,104 @@ endi print ========= step5 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 3) -x s1 s1: -sql insert into d2.t2 values(now, 3) -x s2 -s2: -sql insert into d3.t3 values(now, 3) -x s3 -s3: -sql insert into d4.t4 values(now, 3) -x s4 -s4: +#sql insert into d2.t2 values(now, 3) -x s2 +#s2: +#sql insert into d3.t3 values(now, 3) -x s3 +#s3: +#sql insert into d4.t4 values(now, 3) -x s4 +#s4: print ========= step6 system sh/exec.sh -n dnode2 -s start -sleep 5000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 -sql insert into d1.t1 values(now, 4) -x s5 -s5: +#sql insert into d1.t1 values(now, 4) -x s5 +#s5: sql insert into d2.t2 values(now, 4) -x s6 s6: -sql insert into d3.t3 values(now, 4) -x s7 -s7: -sql insert into d4.t4 values(now, 4) -x s8 -s8: +#sql insert into d3.t3 values(now, 4) -x s7 +#s7: +#sql insert into d4.t4 values(now, 4) -x s8 +#s8: print ========= step7 system sh/exec.sh -n dnode3 -s start -sleep 5000 + +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step7 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) sql insert into d3.t3 values(now, 5) sql insert into d4.t4 values(now, 5) -sleep 1000 sql select * from d1.t1 sql select * from d2.t2 @@ -148,9 +303,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce21.sim b/tests/script/unique/db/replica_reduce21.sim index 7fe7d9504a566789a4a4470d5a4cffe60b7f68c9..d3a76485f81616497a8c4859aafefda7ef371c01 100644 --- a/tests/script/unique/db/replica_reduce21.sim +++ b/tests/script/unique/db/replica_reduce21.sim @@ -14,12 +14,42 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 2 @@ -63,12 +93,25 @@ sql create database d5 replica 1 print ========= step3 alter d1 sql alter database d1 replica 1 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + print ========= step4 query d1 sql insert into d1.t1 values(now, 2) sql select * from d1.t1 -sleep 1000 if $rows != 2 then return -1 endi @@ -77,22 +120,20 @@ print ========= step5 query d5 sql create table d5.t5 (ts timestamp, i int) sql insert into d5.t5 values(now, 1); sql select * from d5.t5 -sleep 1000 if $rows != 1 then return -1 endi return print ========= step7 drop d1 sql drop database d1 -sleep 5000 - sql reset query cache +sleep 100 + print ========= step8 sql insert into d5.t5 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d5.t5 if $rows != 2 then @@ -116,13 +157,11 @@ endi print ======== step9 stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 sql insert into d5.t5 values(now, 3) sql insert into d2.t2 values(now, 3) sql insert into d3.t3 values(now, 3) sql insert into d4.t4 values(now, 3) -sleep 1000 sql select * from d5.t5 if $rows != 3 then @@ -145,10 +184,4 @@ if $rows != 3 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce31.sim b/tests/script/unique/db/replica_reduce31.sim index cbe0417dbd995644099d76a70beaeccb17723129..2313cbd85e05813a29bce15f96ac0094a3eb1664 100644 --- a/tests/script/unique/db/replica_reduce31.sim +++ b/tests/script/unique/db/replica_reduce31.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,38 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -38,7 +74,6 @@ sql insert into d1.t1 values(now, 1) sql insert into d2.t2 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -64,27 +99,104 @@ print ========= step2 alter db sql_error alter database d1 replica 1 sql_error alter database d2 replica 1 sql_error alter database d3 replica 1 +sql_error alter database d4 replica 1 sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 +sql alter database d4 replica 2 + +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi -sleep 8000 +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi sql alter database d1 replica 1 sql alter database d2 replica 1 sql alter database d3 replica 1 +sql alter database d4 replica 1 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi -sleep 8000 +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 1 then + goto a1 +endi + +sql show dnodes +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 + +if $data02 != 0 then + goto a1 +endi +if $data12 != 2 then + goto a1 +endi +if $data22 != 2 then + goto a1 +endi print ========= step3 sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -110,13 +222,44 @@ print ========= step4 alter db sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 -sleep 8000 +sql alter database d4 replica 2 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step4 +endi sql insert into d1.t1 values(now, 3) sql insert into d2.t2 values(now, 3) sql insert into d3.t3 values(now, 3) sql insert into d4.t4 values(now, 3) -sleep 1000 sql select * from d1.t1 if $rows != 3 then @@ -140,10 +283,8 @@ endi print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 sql insert into d1.t1 values(now, 4) -x step1 step1: @@ -156,9 +297,7 @@ step4: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 sql insert into d1.t1 values(now, 5) -x step5 step5: @@ -171,13 +310,11 @@ step8: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 sql insert into d1.t1 values(now, 6) sql insert into d2.t2 values(now, 6) sql insert into d3.t3 values(now, 6) sql insert into d4.t4 values(now, 6) -sleep 1000 sql select * from d1.t1 sql select * from d2.t2 @@ -187,8 +324,4 @@ sql select * from d4.t4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/db/replica_reduce32.sim b/tests/script/unique/db/replica_reduce32.sim index c453f16ce652bcc0c9522e39ab9016c0028180dd..a5eb78dacb6065413e06a50d7a0d9f7e28897d9f 100644 --- a/tests/script/unique/db/replica_reduce32.sim +++ b/tests/script/unique/db/replica_reduce32.sim @@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode1 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 1 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 1 + print ========= start dnodes system sh/exec.sh -n dnode1 -s start sql connect @@ -21,7 +26,43 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != null then + goto step1 +endi +if $data2_3 != null then + goto step1 +endi print ======== step1 sql create database d1 replica 3 @@ -38,7 +79,6 @@ sql insert into d2.t2 values(now, 1) sql insert into d1.t1 values(now, 1) sql insert into d3.t3 values(now, 1) sql insert into d4.t4 values(now, 1) -sleep 1000 sql select * from d1.t1 if $rows != 1 then @@ -65,14 +105,44 @@ sql alter database d1 replica 2 sql alter database d2 replica 2 sql alter database d3 replica 2 sql alter database d4 replica 2 -sleep 12000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi print ========= step3 sql insert into d1.t1 values(now, 2) sql insert into d2.t2 values(now, 2) sql insert into d3.t3 values(now, 2) sql insert into d4.t4 values(now, 2) -sleep 1000 sql select * from d1.t1 if $rows != 2 then @@ -95,40 +165,111 @@ if $rows != 2 then endi sql reset query cache -sleep 1000 +sleep 200 print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep 5000 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql insert into d1.t1 values(now, 3) -x s41 +s41: +sql insert into d2.t2 values(now, 3) -x s42 +s42: +sql insert into d3.t3 values(now, 3) -x s43 +s43: +sql insert into d4.t4 values(now, 3) -x s44 +s44: -sql_error select * from d1.t1 -sql_error select * from d2.t2 -sql_error select * from d3.t3 -sql_error select * from d4.t4 +sql select * from d1.t1 -x s45 +s45: +sql select * from d2.t2 -x s46 +s46: +sql select * from d3.t3 -x s47 +s47: +sql select * from d4.t4 -x s48 +s48: print ========= step5 system sh/exec.sh -n dnode2 -s start -sleep 5000 -system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step5 +endi +system sh/exec.sh -n dnode3 -s stop -x SIGINT sql reset query cache -sleep 1000 +sleep 100 -sql_error insert into d1.t1 values(now, 3) -sql_error insert into d2.t2 values(now, 3) -sql_error insert into d3.t3 values(now, 3) -sql_error insert into d4.t4 values(now, 3) +sql_error insert into d1.t1 values(now, 3) -x s51 +s51: +sql_error insert into d2.t2 values(now, 3) -x s52 +s52: +sql_error insert into d3.t3 values(now, 3) -x s53 +s53: +sql_error insert into d4.t4 values(now, 3) -x s54 +s54: print ========= step6 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show d1.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d2.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d3.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi + +sql show d4.vgroups +print online vnodes $data03 +if $data03 != 2 then + goto step6 +endi sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) @@ -150,9 +291,4 @@ print d4.t4 $rows system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/balance1.sim b/tests/script/unique/dnode/balance1.sim index 10d185f3536c1c7b4c61bc9401655d058ae17e88..8743204a03e06d0aefc66b2f70cbdc111e317237 100644 --- a/tests/script/unique/dnode/balance1.sim +++ b/tests/script/unique/dnode/balance1.sim @@ -33,7 +33,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 sql create database d1 sql create table d1.t1 (t timestamp, i int) @@ -50,15 +49,14 @@ if $data2_1 != 1 then endi print ========== step2 -sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -124,8 +122,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -174,8 +172,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -258,7 +256,7 @@ endi system sh/exec.sh -n dnode3 -s stop -x SIGINT sql reset query cache -sleep 1000 +sleep 100 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance2.sim b/tests/script/unique/dnode/balance2.sim index 23897df690634759de027c9b26ad0449694242a9..7b07eb3f37334e78d1b2828120472d4ea9ebb5a1 100644 --- a/tests/script/unique/dnode/balance2.sim +++ b/tests/script/unique/dnode/balance2.sim @@ -32,7 +32,29 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database d1 replica 2 sql create table d1.t1 (t timestamp, i int) @@ -70,8 +92,8 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -90,7 +112,6 @@ if $data2_3 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step3 sql create dnode $hostname4 @@ -156,8 +177,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -189,8 +210,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -217,10 +238,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/balance3.sim b/tests/script/unique/dnode/balance3.sim index 9f7f3abb8bde66fdad63351d76e6d11e64db38aa..f5558d504e34608eafb2a076a8739024e13dcff2 100644 --- a/tests/script/unique/dnode/balance3.sim +++ b/tests/script/unique/dnode/balance3.sim @@ -38,7 +38,32 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi sql create database d1 replica 3 sql create table d1.t1 (t timestamp, i int) @@ -81,7 +106,7 @@ sql drop dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 + sleep 1000 if $x == 20 then return -1 endi @@ -106,7 +131,6 @@ if $data2_4 != 2 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step sql create dnode $hostname5 system sh/exec.sh -n dnode5 -s start @@ -114,8 +138,8 @@ system sh/exec.sh -n dnode5 -s start $x = 0 show3: $x = $x + 1 - sleep 4000 - if $x == 15 then + sleep 1000 + if $x == 60 then return -1 endi @@ -154,8 +178,8 @@ sql insert into d3.t3 values(now+5s, 31) $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -189,8 +213,8 @@ system sh/exec.sh -n dnode6 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -216,8 +240,8 @@ sql drop dnode $hostname3 $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 20 then + sleep 1000 + if $x == 30 then return -1 endi @@ -245,10 +269,8 @@ if $data2_5 != 3 then endi system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 - sql reset query cache -sleep 1000 +sleep 100 print ========== step7 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/dnode/lossdata.sim b/tests/script/unique/dnode/lossdata.sim new file mode 100644 index 0000000000000000000000000000000000000000..89ba7169708eedb21c663a5e4fe0f897d42c5f43 --- /dev/null +++ b/tests/script/unique/dnode/lossdata.sim @@ -0,0 +1,165 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 2 +sql create table d1.t1 (t timestamp, i int) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 1 then + return -1 +endi +if $data2_3 != 1 then + return -1 +endi + +print ========== step3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi +if $data2_4 != 0 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step4 +sql drop dnode $hostname3 + +$i = 0 +$rowNum = 10000 + +while $i < $rowNum + $ts = 1500000000000 + $i + sql insert into d1.t1 values( $ts , $i ) + + $i = $i + 1 +endw + +print insert $rowNum finished + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 1 then + goto show4 +endi +if $data2_3 != null then + goto show4 +endi +if $data2_4 != 1 then + goto show4 +endi + +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print ========== step5 +sql select count(*) from d1.t1 +print select count(*) from d1.t1 ==> $data00 +if $data00 != $rowNum then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/m2.sim b/tests/script/unique/dnode/m2.sim new file mode 100644 index 0000000000000000000000000000000000000000..5fdf3b740081e74024f02616483af2c943d47eb9 --- /dev/null +++ b/tests/script/unique/dnode/m2.sim @@ -0,0 +1,367 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 2 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 15) +sql insert into d1.t1 values(now+2s, 14) +sql insert into d1.t1 values(now+3s, 13) +sql insert into d1.t1 values(now+4s, 12) +sql insert into d1.t1 values(now+5s, 11) + +sql create database d2 replica 2 +sql create table d2.t2 (t timestamp, i int) +sql insert into d2.t2 values(now+1s, 25) +sql insert into d2.t2 values(now+2s, 24) +sql insert into d2.t2 values(now+3s, 23) +sql insert into d2.t2 values(now+4s, 22) +sql insert into d2.t2 values(now+5s, 21) + +sql create database d3 replica 2 +sql create table d3.t3 (t timestamp, i int) +sql insert into d3.t3 values(now+1s, 35) +sql insert into d3.t3 values(now+2s, 34) +sql insert into d3.t3 values(now+3s, 33) +sql insert into d3.t3 values(now+4s, 32) +sql insert into d3.t3 values(now+5s, 31) + +sql create database d4 replica 2 +sql create table d4.t4 (t timestamp, i int) +sql insert into d4.t4 values(now+1s, 45) +sql insert into d4.t4 values(now+2s, 44) +sql insert into d4.t4 values(now+3s, 43) +sql insert into d4.t4 values(now+4s, 42) +sql insert into d4.t4 values(now+5s, 41) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 4 then + return -1 +endi +if $data2_3 != 4 then + return -1 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start +sql create dnode $hostname5 +system sh/exec.sh -n dnode5 -s start + + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 2 then + goto show3 +endi +if $data2_3 != 2 then + goto show3 +endi +if $data2_4 != 2 then + goto show3 +endi +if $data2_5 != 2 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show3 +endi + +print ========== step4 +sql drop dnode $hostname2 + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show4 +endi + +sql reset query cache +sleep 100 +system sh/exec.sh -n dnode2 -s stop -x SIGINT + +print ========== step5 +sql drop dnode $hostname3 + +$x = 0 +show5: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show5 +endi +if $data2_3 != null then + goto show5 +endi +if $data2_4 != 4 then + goto show5 +endi +if $data2_5 != 4 then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data03 != 2 then + goto show5 +endi + +sql reset query cache +sleep 100 +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print ========== step6 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 11 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data41 != 15 then + return -1 +endi + +sql select * from d2.t2 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 21 then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data21 != 23 then + return -1 +endi +if $data31 != 24 then + return -1 +endi +if $data41 != 25 then + return -1 +endi + +sql select * from d3.t3 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +sql select * from d4.t4 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 41 then + return -1 +endi +if $data11 != 42 then + return -1 +endi +if $data21 != 43 then + return -1 +endi +if $data31 != 44 then + return -1 +endi +if $data41 != 45 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/m3.sim b/tests/script/unique/dnode/m3.sim new file mode 100644 index 0000000000000000000000000000000000000000..5850147d045d4e58fc8b700bcbb7d70318f0420a --- /dev/null +++ b/tests/script/unique/dnode/m3.sim @@ -0,0 +1,359 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode5 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 +system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +sql create dnode $hostname4 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +print ========== step2 + +sql create database d1 replica 3 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 15) +sql insert into d1.t1 values(now+2s, 14) +sql insert into d1.t1 values(now+3s, 13) +sql insert into d1.t1 values(now+4s, 12) +sql insert into d1.t1 values(now+5s, 11) + +sql create database d2 replica 3 +sql create table d2.t2 (t timestamp, i int) +sql insert into d2.t2 values(now+1s, 25) +sql insert into d2.t2 values(now+2s, 24) +sql insert into d2.t2 values(now+3s, 23) +sql insert into d2.t2 values(now+4s, 22) +sql insert into d2.t2 values(now+5s, 21) + +sql create database d3 replica 3 +sql create table d3.t3 (t timestamp, i int) +sql insert into d3.t3 values(now+1s, 35) +sql insert into d3.t3 values(now+2s, 34) +sql insert into d3.t3 values(now+3s, 33) +sql insert into d3.t3 values(now+4s, 32) +sql insert into d3.t3 values(now+5s, 31) + +sql create database d4 replica 3 +sql create table d4.t4 (t timestamp, i int) +sql insert into d4.t4 values(now+1s, 45) +sql insert into d4.t4 values(now+2s, 44) +sql insert into d4.t4 values(now+3s, 43) +sql insert into d4.t4 values(now+4s, 42) +sql insert into d4.t4 values(now+5s, 41) + +print ========== step2.1 + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 + +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 4 then + return -1 +endi +if $data2_3 != 4 then + return -1 +endi +if $data2_4 != 4 then + return -1 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 3 then + return -1 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 2 then + return -1 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi + +print ========== step3 +sql create dnode $hostname5 +system sh/exec.sh -n dnode5 -s start + +$x = 0 +show3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != 3 then + goto show3 +endi +if $data2_3 != 3 then + goto show3 +endi +if $data2_4 != 3 then + goto show3 +endi +if $data2_5 != 3 then + goto show3 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 2 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show3 +endi + +print ========== step4 +sql drop dnode $hostname2 + +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +print dnode5 openVnodes $data2_5 + +if $data2_2 != null then + goto show4 +endi +if $data2_3 != 4 then + goto show4 +endi +if $data2_4 != 4 then + goto show4 +endi +if $data2_5 != 4 then + goto show4 +endi + +sql show d1.vgroups; +print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 5 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d2.vgroups; +print d2.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 3 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d3.vgroups; +print d3.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql show d4.vgroups; +print d4.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $data04 != 4 then + return -1 +endi +if $data03 != 3 then + goto show4 +endi + +sql reset query cache +sleep 100 + +print ========== step5 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 11 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data41 != 15 then + return -1 +endi + +sql select * from d2.t2 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 21 then + return -1 +endi +if $data11 != 22 then + return -1 +endi +if $data21 != 23 then + return -1 +endi +if $data31 != 24 then + return -1 +endi +if $data41 != 25 then + return -1 +endi + +sql select * from d3.t3 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +sql select * from d4.t4 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 41 then + return -1 +endi +if $data11 != 42 then + return -1 +endi +if $data21 != 43 then + return -1 +endi +if $data31 != 44 then + return -1 +endi +if $data41 != 45 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/offline3.sim b/tests/script/unique/dnode/offline3.sim new file mode 100644 index 0000000000000000000000000000000000000000..93c75e3b13333d55aea7cb2417413a14a1c13e62 --- /dev/null +++ b/tests/script/unique/dnode/offline3.sim @@ -0,0 +1,111 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode2 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode3 -c offlineThreshold -v 3 +system sh/cfg.sh -n dnode4 -c offlineThreshold -v 3 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 300 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 300 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +if $data2_1 != master then + goto step1 +endi + +print ========== step2 +sql create database db replica 3 +sql use db +sql create table tb (ts timestamp, value int) +sql insert into tb values (now, 1) +sql insert into tb values (now, 2) + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +print ========== step2 +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show vgroups; +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi +if $data4_3 != ready then + goto step2 +endi +if $data4_4 != null then + goto step2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/dnode/reason.sim b/tests/script/unique/dnode/reason.sim index ad61a81b9782cf3d52c65059100c879423ae4843..c685b1129d924131d38e60369d97924250b3ad66 100644 --- a/tests/script/unique/dnode/reason.sim +++ b/tests/script/unique/dnode/reason.sim @@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode2 -i 2 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -18,38 +17,75 @@ endi print ========== step2 system sh/exec.sh -n dnode2 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes -print dnode1 off: $data7_1 -print dnode2 off: $data7_2 +print dnode1 $data4_1 +print dnode2 $data4_2 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -sleep 3000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_2 if $data7_2 != @status msg timeout@ then - return -1 + goto step3 endi print ========== step4 sql drop dnode $hostname2 -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes if $rows != 1 then - return -1 + goto step4 endi print ========== step5 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode2 off: $data7_3 if $data7_3 != @dnodeId not match@ then - return -1 + goto step5 endi print ========== step6 @@ -58,12 +94,19 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 3 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 3000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode4 off: $data7_4 if $data7_4 != @mnEqualVn not match@ then - return -1 + goto step6 endi print ========== step7 @@ -72,12 +115,19 @@ system sh/cfg.sh -n dnode5 -c statusInterval -v 3 system sh/exec.sh -n dnode5 -s start sql create dnode $hostname5 -sleep 3000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode5 off: $data7_5 if $data7_5 != @interval not match@ then - return -1 + goto step7 endi print ========== step8 @@ -86,12 +136,19 @@ system sh/cfg.sh -n dnode6 -c balance -v 0 system sh/exec.sh -n dnode6 -s start sql create dnode $hostname6 -sleep 3000 +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode6 off: $data7_6 if $data7_6 != @balance not match@ then - return -1 + goto step8 endi print ========== step9 @@ -100,12 +157,19 @@ system sh/cfg.sh -n dnode7 -c maxTablesPerVnode -v 3000 system sh/exec.sh -n dnode7 -s start sql create dnode $hostname7 -sleep 3000 +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode7 off: $data7_7 if $data7_7 != @maxTabPerVn not match@ then - return -1 + goto step9 endi print ========== step10 @@ -114,12 +178,19 @@ system sh/cfg.sh -n dnode8 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode8 -s start sql create dnode $hostname8 -sleep 3000 +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + sql show dnodes print dnode1 off: $data7_1 print dnode8 off: $data7_8 if $data7_8 != @maxVgPerDb not match@ then - return -1 + goto step10 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/dnode/remove1.sim b/tests/script/unique/dnode/remove1.sim index 6bde68b8b5c7c5c7050141645fc689f08a5fe328..6f830d2cf8d50975a494854de9932fa74f41fb5c 100644 --- a/tests/script/unique/dnode/remove1.sim +++ b/tests/script/unique/dnode/remove1.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,12 +99,11 @@ endi print ========== step3 sql drop dnode $hostname2 -sleep 7001 $x = 0 show3: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -101,8 +118,8 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -115,7 +132,6 @@ if $data2_2 != null then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 print ========== step5 sql create dnode $hostname4 @@ -124,8 +140,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show5: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index 903e262be79a73e76b664c077b4faa0e90139262..f97e55164b14bc62237513a057cff21c36073179 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create database d1 @@ -50,7 +49,26 @@ endi print ========== step2 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 9000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi sql create database d3 replica 2 sql create table d3.t3 (t timestamp, i int) @@ -81,7 +99,6 @@ endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT sql drop dnode $hostname2 -sleep 5000 sql show dnodes print dnode1 openVnodes $data2_1 @@ -91,14 +108,20 @@ print ========== step4 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 if $data2_3 != 0 then - return -1 + goto step4 endi print ============ step 4.1 @@ -107,8 +130,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/dnode/vnode_clean.sim b/tests/script/unique/dnode/vnode_clean.sim index f90f3d3fd010c7ac08305c3b38b12c7b48f5a70b..76311a6cacb2d2c06ff4125b689d3782c3e0dc98 100644 --- a/tests/script/unique/dnode/vnode_clean.sim +++ b/tests/script/unique/dnode/vnode_clean.sim @@ -45,8 +45,8 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -86,8 +86,8 @@ sql drop dnode $hostname2 $x = 0 show4: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -106,15 +106,14 @@ endi system sh/exec.sh -n dnode2 -s stop -x SIGINT print ========== step5 -sleep 5000 sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start $x = 0 show5: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi sql show dnodes @@ -153,8 +152,8 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show7: $x = $x + 1 - sleep 3000 - if $x == 20 then + sleep 1000 + if $x == 40 then return -1 endi @@ -184,8 +183,8 @@ sql insert into d4.t4 values(now+5s, 41) $x = 0 show8: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi sql show dnodes @@ -208,8 +207,8 @@ sql drop dnode $hostname3 $x = 0 show9: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -228,7 +227,6 @@ if $data2_4 != 4 then endi system sh/exec.sh -n dnode3 -s stop -sleep 5000 print ========== step10 sql select * from d1.t1 order by t desc diff --git a/tests/script/unique/import/replica2.sim b/tests/script/unique/import/replica2.sim index f2836fe470562dd6229a9db7f694c7cfe7826cb3..54ce28543e741a1264e1bcf76c7152bee74564c0 100644 --- a/tests/script/unique/import/replica2.sim +++ b/tests/script/unique/import/replica2.sim @@ -27,11 +27,30 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi sql create database ir2db replica 2 days 7 sql use ir2db @@ -96,9 +115,22 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a1: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a1 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -163,9 +195,22 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +a2: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a2 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -193,9 +238,20 @@ endi print ================= step13 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +$x = 0 +a3: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a3 +endi print ================= step14 #1520000000000 @@ -214,12 +270,24 @@ endi print ================= step15 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 2 then + goto a4 +endi + +sql select * from tb; if $rows != 52 then - return -1 + goto a4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/unique/import/replica3.sim b/tests/script/unique/import/replica3.sim index 3a9f03a7ea5140938b21b8f5c9dc983eb701c2c2..714141c412793f6608b11beb0a455ee3d37a3310 100644 --- a/tests/script/unique/import/replica3.sim +++ b/tests/script/unique/import/replica3.sim @@ -27,7 +27,6 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql create dnode $hostname2 @@ -35,6 +34,29 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi sql create database ir3db replica 3 days 7 sql use ir3db @@ -99,9 +121,20 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +$x = 0 +a4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto a4 +endi + sql select * from tb; if $rows != 14 then return -1 @@ -166,9 +199,21 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode1 -s start -sleep 5000 + +$x = 0 +step10: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step10 +endi + sql select * from tb; print $rows if $rows != 35 then @@ -217,10 +262,21 @@ endi print ================= step15 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +$x = 0 +step15: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi +sql show vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step15 +endi +sql select * from tb; if $rows != 52 then return -1 endi diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index 12f0013191b90cab8b85c9050f25ce182e1807b4..9b02933cbfc98c12c06ad1918d09dea9aa05ded0 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -194,36 +194,35 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 - -if $d2v2status != master then +if $data5_4 != master then + print $data5_4 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then +print $data5_2 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then +print $data5_3 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then + print $data7_4 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then + print $data7_3 sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then + print $data7_2 sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -264,4 +263,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index 29da1fd77359146b6fb44fa8347e7b850947558d..90183949e7f516a5fb1c2127652217fefcc36d8d 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -196,36 +196,29 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 - -if $d2v2status != master then +if $data5_4 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -266,4 +259,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index 8a1132de2e62a6d97d37648435dd94d748a74c28..02e2cd02e1523df7ed0b1d2b29aade60585cd637 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -194,36 +194,29 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data5_4 -$d2v3status = $data5_2 -$d2v4status = $data5_3 -$d1v2status = $data7_4 -$d1v3status = $data7_2 -$d1v4status = $data7_3 - -if $d2v2status != master then +if $data5_4 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v3status != master then +if $data5_3 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d2v4status != master then +if $data5_2 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v2status != slave then +if $data7_4 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v3status != slave then +if $data7_3 != master then sleep 2000 goto wait_dnode1_vgroup_slave endi -if $d1v4status != slave then +if $data7_2 != slave then sleep 2000 goto wait_dnode1_vgroup_slave endi @@ -264,4 +257,13 @@ sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then return -1 -endi \ No newline at end of file +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt20.sim b/tests/script/unique/mnode/mgmt20.sim index e51d4299255248f3fb19371183536e5e62123655..ee9c2b914f237f619f342122a3b1fd2bccc5cf67 100644 --- a/tests/script/unique/mnode/mgmt20.sim +++ b/tests/script/unique/mnode/mgmt20.sim @@ -11,7 +11,6 @@ system sh/cfg.sh -n dnode2 -c monitor -v 1 print ============== step1 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 3000 sql connect print ============== step2 @@ -20,8 +19,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -51,6 +50,24 @@ $d1_first = $rows sql select * from log.dn2 $d2_first = $rows +$x = 0 +show4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + goto show4 +endi +if $data2_2 != slave then + goto show4 +endi + sleep 3000 sql select * from log.dn1 $d1_second = $rows diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index d55e36d7fc627c3dcfb3e9c21c9396960a0c9287..1dc419b17dbd537e09833780c0104e304f8b71fe 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -9,7 +9,6 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -26,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -64,8 +63,8 @@ sql connect $x = 0 show6: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -82,13 +81,12 @@ endi print ============== step7 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 5000 $x = 0 show7: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi diff --git a/tests/script/unique/mnode/mgmt23.sim b/tests/script/unique/mnode/mgmt23.sim index d1820ef8c6a886fdbc55bda6d7b9e79a0ea51e1b..19c7b4ba762d4bf5a73c10c1afa39e927c7a1c91 100644 --- a/tests/script/unique/mnode/mgmt23.sim +++ b/tests/script/unique/mnode/mgmt23.sim @@ -25,8 +25,8 @@ sql create dnode $hostname2 $x = 0 show2: $x = $x + 1 - sleep 2000 - if $x == 10 then + sleep 1000 + if $x == 20 then return -1 endi @@ -65,7 +65,14 @@ endi print ============== step4 sql drop dnode $hostname2 -sleep 10000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -76,13 +83,13 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop @@ -93,7 +100,14 @@ sql create dnode $hostname2 system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/exec.sh -n dnode2 -s start -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -104,19 +118,17 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 - sql_error show mnodes print ============== step7 @@ -126,7 +138,4 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode5 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt30.sim b/tests/script/unique/mnode/mgmt30.sim new file mode 100644 index 0000000000000000000000000000000000000000..a9488799334ef81c82e231bc7782993b7a33a4fa --- /dev/null +++ b/tests/script/unique/mnode/mgmt30.sim @@ -0,0 +1,68 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 3000 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 3000 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 3000 + +print ============== step1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +print dnode3 ==> $data3_3 +if $data2_1 != master then + return -1 +endi +if $data3_2 != null then + return -1 +endi +if $data3_3 != null then + return -1 +endi + +print ============== step2 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 5000 + +sql create dnode $hostname2 +sql create dnode $hostname3 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes +$dnode1Role = $data2_1 +$dnode2Role = $data2_2 +$dnode3Role = $data2_3 +print dnode1 ==> $dnode1Role +print dnode2 ==> $dnode2Role +print dnode3 ==> $dnode3Role + +if $dnode1Role != master then + goto step2 +endi +if $dnode2Role != slave then + goto step2 +endi +if $dnode3Role != slave then + goto step2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/mgmt33.sim b/tests/script/unique/mnode/mgmt33.sim index 205e1b05d376991d72d1676262ed50c4203d3bf3..ce7cdce35d8c0463564f46d26a0711d39340c8bf 100644 --- a/tests/script/unique/mnode/mgmt33.sim +++ b/tests/script/unique/mnode/mgmt33.sim @@ -28,7 +28,14 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -39,19 +46,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -62,18 +76,25 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi print ============== step4 sql drop dnode $hostname2 -sleep 8000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -84,25 +105,30 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != null then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step5 sql create dnode $hostname2 -sleep 8000 + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -113,20 +139,26 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != slave then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi print ============== step6 system sh/exec.sh -n dnode1 -s stop -sleep 10000 - -sql show mnodes +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step6 $dnode1Role = $data2_1 $dnode2Role = $data2_4 $dnode3Role = $data2_3 @@ -135,7 +167,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != offline then - return -1 + goto step6 endi #if $dnode2Role != master then # return -1 @@ -146,9 +178,15 @@ endi print ============== step7 sql drop dnode $hostname1 -sleep 8000 - -sql show mnodes +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -157,7 +195,7 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != null then - return -1 + goto step7 endi #if $dnode2Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmt34.sim b/tests/script/unique/mnode/mgmt34.sim index 96e3133d762362cc47f656421dfab8470e37cb2e..d8a46b0955f59273279bbbc5c89c07c05db672d7 100644 --- a/tests/script/unique/mnode/mgmt34.sim +++ b/tests/script/unique/mnode/mgmt34.sim @@ -12,7 +12,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect sql show mnodes @@ -32,7 +31,13 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 8000 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -45,22 +50,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step2 endi if $dnode2Role != slave then - return -1 + goto step2 endi if $dnode3Role != null then - return -1 + goto step2 endi if $dnode4Role != null then - return -1 + goto step2 endi print ============== step3 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 8000 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -73,23 +85,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step3 endi if $dnode2Role != slave then - return -1 + goto step3 endi if $dnode3Role != slave then - return -1 + goto step3 endi if $dnode4Role != null then - return -1 + goto step3 endi print ============== step4 system sh/exec.sh -n dnode4 -s start sql create dnode $hostname4 -sleep 8000 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -102,21 +120,27 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != slave then - return -1 + goto step4 endi if $dnode4Role != null then - return -1 + goto step4 endi print ============== step5 sql drop dnode $hostname2 -sleep 8000 +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -129,28 +153,32 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step5 endi if $dnode2Role != null then - return -1 + goto step5 endi if $dnode3Role != slave then - return -1 + goto step5 endi if $dnode4Role != slave then - return -1 + goto step5 endi system sh/exec.sh -n dnode2 -s stop -sleep 3000 - system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/exec.sh -n dnode2 -s start print ============== step6 sql create dnode $hostname2 -sleep 8000 +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi sql show mnodes $dnode1Role = $data2_1 @@ -163,23 +191,29 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != master then - return -1 + goto step6 endi if $dnode2Role != null then - return -1 + goto step6 endi if $dnode3Role != slave then - return -1 + goto step6 endi if $dnode4Role != slave then - return -1 + goto step6 endi print ============== step7 system sh/exec.sh -n dnode1 -s stop -sleep 4000 +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step7 $dnode1Role = $data2_1 $dnode2Role = $data2_2 $dnode3Role = $data2_3 @@ -190,14 +224,19 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != offline then - return -1 + goto step7 endi print ============== step8 sql drop dnode $hostname1 -sleep 8000 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi -sql show mnodes +sql show mnodes -x step8 $dnode1Role = $data2_1 $dnode2Role = $data2_5 $dnode3Role = $data2_3 @@ -208,10 +247,10 @@ print dnode3 ==> $dnode3Role print dnode4 ==> $dnode4Role if $dnode1Role != null then - return -1 + goto step8 endi if $dnode2Role != slave then - return -1 + goto step8 endi #if $dnode3Role != master then # return -1 diff --git a/tests/script/unique/mnode/mgmtr2.sim b/tests/script/unique/mnode/mgmtr2.sim index 1fba912d492785900f88686097074eaafc4445af..0c9f961d253e1ca57477f5612ed5f1f28c7b9e26 100644 --- a/tests/script/unique/mnode/mgmtr2.sim +++ b/tests/script/unique/mnode/mgmtr2.sim @@ -55,6 +55,14 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start print ============== step4 +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi + sql show mnodes $dnode1Role = $data2_1 @@ -65,20 +73,15 @@ print dnode2 ==> $dnode2Role print dnode3 ==> $dnode3Role if $dnode1Role != master then - return -1 + goto step4 endi if $dnode2Role != slave then - return -1 + goto step4 endi if $dnode3Role != null then - return -1 + goto step4 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/vnode/many.sim b/tests/script/unique/vnode/many.sim index 8bd6372ae9c677f5fc8475430ec3135b21409835..869c7f82953e99ef9e14a193f3894c4b53466b9e 100644 --- a/tests/script/unique/vnode/many.sim +++ b/tests/script/unique/vnode/many.sim @@ -23,7 +23,37 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db1 replica 2 diff --git a/tests/script/unique/vnode/replica2_repeat.sim b/tests/script/unique/vnode/replica2_repeat.sim index e862d745d4fac88137d1053b1bccd22d72c5f7cc..5dbb24437d3dc534018b788c5be1e29bf32cb72d 100644 --- a/tests/script/unique/vnode/replica2_repeat.sim +++ b/tests/script/unique/vnode/replica2_repeat.sim @@ -23,7 +23,38 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 2 @@ -64,7 +95,7 @@ print ======== step7 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/script/unique/vnode/replica3_basic.sim b/tests/script/unique/vnode/replica3_basic.sim index ab5bebb50daf8a3be63d039152a020f0fff7000b..0ff42b523b8982c85bd84bb251715585a66137fc 100644 --- a/tests/script/unique/vnode/replica3_basic.sim +++ b/tests/script/unique/vnode/replica3_basic.sim @@ -16,14 +16,48 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi +if $data2_2 != slave then + goto step1 +endi +if $data2_3 != slave then + goto step1 +endi $N = 10 $table = table_r3 $db = db1 -sleep 3000 - print =================== step 1 sql create database $db replica 3 @@ -66,7 +100,21 @@ endi print =================== step 4 system sh/exec.sh -n dnode2 -s start -sleep 2000 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step4 +endi + $y = $x + $N $expect = $N * 3 while $x < $y @@ -83,7 +131,6 @@ endi print =================== step 5 system sh/exec.sh -n dnode3 -s stop -sleep 2000 $y = $x + $N $expect = $N * 4 while $x < $y @@ -100,7 +147,21 @@ endi print =================== step 6 system sh/exec.sh -n dnode3 -s start -sleep 2000 + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step6 +endi + $y = $x + $N $expect = $N * 5 while $x < $y @@ -117,7 +178,6 @@ endi print =================== step 7 system sh/exec.sh -n dnode1 -s stop -sleep 2000 $y = $x + $N $expect = $N * 6 while $x < $y @@ -134,7 +194,21 @@ endi print =================== step 8 system sh/exec.sh -n dnode1 -s start -sleep 2000 + +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 40 then + return -1 + endi + +sql show db1.vgroups +print online vnodes $data03 +if $data03 != 3 then + goto step8 +endi + $y = $x + $N $expect = $N * 7 while $x < $y diff --git a/tests/script/unique/vnode/replica3_repeat.sim b/tests/script/unique/vnode/replica3_repeat.sim index 4b5c852de8a6f841ae75b9716577ae735f970b3f..d866fb05d89b5441d83390a154324fb4290c451d 100644 --- a/tests/script/unique/vnode/replica3_repeat.sim +++ b/tests/script/unique/vnode/replica3_repeat.sim @@ -26,7 +26,42 @@ sql create dnode $hostname4 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start -sleep 3000 + + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 +print dnode4 $data4_4 + +if $data4_1 != ready then + goto step1 +endi +if $data4_2 != ready then + goto step1 +endi +if $data4_3 != ready then + goto step1 +endi +if $data4_4 != ready then + goto step1 +endi + +sql show mnodes +print mnode1 $data2_1 +print mnode1 $data2_2 +print mnode1 $data2_3 +if $data2_1 != master then + goto step1 +endi print ========= step1 sql create database db replica 3 @@ -75,7 +110,7 @@ print ======== step8 $lastRows = $data00 print ======== loop Times $x -if $x < 5 then +if $x < 2 then $x = $x + 1 goto loop endi diff --git a/tests/test-all.sh b/tests/test-all.sh index 4448c8a2a4b86c1f0ada0e924763457fed81a825..19d78032554682d533c89bc4e4bd473689364d87 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -6,7 +6,16 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' - +function git_branch { + branch="`git branch 2>/dev/null | grep "^\*" | sed -e "s/^\*\ //"`" + if [ "${branch}" != "" ];then + if [ "${branch}" = "(no branch)" ];then + branch="(`git rev-parse --short HEAD`...)" + fi + branch=(${branch////_}) + echo "$branch" + fi +} function runSimCaseOneByOne { while read -r line; do if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then @@ -31,13 +40,24 @@ function runSimCaseOneByOnefq { case=`echo $line | grep sim$ |awk '{print $NF}'` start_time=`date +%s` - ./test.sh -f $case > /dev/null 2>&1 && \ + IN_TDINTERNAL="community" + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + ./test.sh -f $case > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ ( grep 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log - - + else + ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ + ( grep 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log + fi + out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cp -r ../../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S"` + else + cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` + fi exit 8 fi end_time=`date +%s` @@ -89,6 +109,7 @@ function runPyCaseOneByOnefq { end_time=`date +%s` out_log=`tail -1 pytest-out.log ` if [[ $out_log =~ 'failed' ]];then + cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` exit 8 fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log @@ -196,10 +217,10 @@ if [ "$2" != "sim" ]; then runPyCaseOneByOnefq fulltest.sh elif [ "$1" == "p1" ]; then echo "### run Python_1 test ###" - runPyCaseOneByOne pytest_1.sh + runPyCaseOneByOnefq pytest_1.sh elif [ "$1" == "p2" ]; then echo "### run Python_2 test ###" - runPyCaseOneByOne pytest_2.sh + runPyCaseOneByOnefq pytest_2.sh elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then exit $(($totalFailed + $totalPyFailed)) elif [ "$1" == "smoke" ] || [ -z "$1" ]; then