diff --git a/Jenkinsfile b/Jenkinsfile index 72882f9891fa148aed927871187174298d3dfe17..6e6daf5c59f29dc678d4f4485ff77d7e2f257fae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,9 +1,7 @@ import hudson.model.Result import hudson.model.*; import jenkins.model.CauseOfInterruption -properties([pipelineTriggers([githubPush()])]) node { - git url: 'https://github.com/taosdata/TDengine.git' } def skipbuild=0 @@ -181,9 +179,9 @@ def pre_test_win(){ git clean -dfx mkdir debug cd debug - call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64 + call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" amd64 cmake ../ -G "NMake Makefiles" - nmake || exit 8 + set CL=/MP nmake nmake || exit 8 nmake install || exit 8 xcopy /e/y/i/f C:\\workspace\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 || exit 8 cd C:\\workspace\\TDinternal\\community\\src\\connector\\python @@ -194,6 +192,7 @@ def pre_test_win(){ } pipeline { agent none + options { skipDefaultCheckout() } environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDinternal/community' @@ -201,67 +200,67 @@ pipeline { stages { stage('pre_build'){ agent{label 'master'} - when { - changeRequest() + options { skipDefaultCheckout() } + when{ + changeRequest() } steps { script{ abort_previous() abortPreviousBuilds() } - sh''' - rm -rf ${WORKSPACE}.tes - cp -r ${WORKSPACE} ${WORKSPACE}.tes - cd ${WORKSPACE}.tes - git fetch - ''' - script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - git checkout 2.0 - ''' - } - else{ - sh ''' - git checkout develop - ''' - } - } - sh''' - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD - ''' + // sh''' + // rm -rf ${WORKSPACE}.tes + // cp -r ${WORKSPACE} ${WORKSPACE}.tes + // cd ${WORKSPACE}.tes + // git fetch + // ''' + // script { + // if (env.CHANGE_TARGET == 'master') { + // sh ''' + // git checkout master + // ''' + // } + // else if(env.CHANGE_TARGET == '2.0'){ + // sh ''' + // git checkout 2.0 + // ''' + // } + // else{ + // sh ''' + // git checkout develop + // ''' + // } + // } + // sh''' + // git fetch origin +refs/pull/${CHANGE_ID}/merge + // git checkout -qf FETCH_HEAD + // ''' - script{ - skipbuild='2' - skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) - println skipbuild - } - sh''' - rm -rf ${WORKSPACE}.tes - ''' + // script{ + // skipbuild='2' + // skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) + // println skipbuild + // } + // sh''' + // rm -rf ${WORKSPACE}.tes + // ''' + // } } } stage('Parallel test stage') { //only build pr + options { skipDefaultCheckout() } when { allOf{ changeRequest() - expression{ - return skipbuild.trim() == '2' - } + not{ expression { env.CHANGE_BRANCH =~ /docs\// }} } } parallel { stage('python_1_s1') { agent{label " slave1 || slave11 "} steps { - pre_test() timeout(time: 55, unit: 'MINUTES'){ sh ''' @@ -414,7 +413,7 @@ pipeline { stage('test_b4_s7') { agent{label " slave7 || slave17 "} steps { - timeout(time: 55, unit: 'MINUTES'){ + timeout(time: 105, unit: 'MINUTES'){ pre_test() sh ''' date @@ -471,35 +470,35 @@ pipeline { } } - // stage('build'){ - // agent{label " wintest "} - // steps { - // pre_test() - // script{ - // while(win_stop == 0){ - // sleep(1) - // } - // } - // } - // } - // stage('test'){ - // agent{label "win"} - // steps{ + stage('build'){ + agent{label " wintest "} + steps { + pre_test() + script{ + while(win_stop == 0){ + sleep(1) + } + } + } + } + stage('test'){ + agent{label "win"} + steps{ - // catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - // pre_test_win() - // timeout(time: 20, unit: 'MINUTES'){ - // bat''' - // cd C:\\workspace\\TDinternal\\community\\tests\\pytest - // .\\test-all.bat Wintest - // ''' - // } - // } - // script{ - // win_stop=1 - // } - // } - // } + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + pre_test_win() + timeout(time: 20, unit: 'MINUTES'){ + bat''' + cd C:\\workspace\\TDinternal\\community\\tests\\pytest + .\\test-all.bat wintest + ''' + } + } + script{ + win_stop=1 + } + } + } } diff --git a/cmake/define.inc b/cmake/define.inc index 10134a94d2e5d40b7528af1ca205105d3235c6d2..e0cdfd3efc6be2673dc60a53f035e132f5a20a55 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -124,17 +124,25 @@ IF (TD_APLHINE) MESSAGE(STATUS "aplhine is defined") ENDIF () -IF (TD_LINUX) - IF (TD_ARM_32) - SET(TD_BUILD_HTTP TRUE) - ADD_DEFINITIONS(-DHTTP_EMBEDDED) - ELSE () - IF (TD_BUILD_HTTP) - ADD_DEFINITIONS(-DHTTP_EMBEDDED) +MESSAGE("before BUILD_HTTP: " ${BUILD_HTTP}) +IF ("${BUILD_HTTP}" STREQUAL "") + IF (TD_LINUX) + IF (TD_ARM_32) + SET(BUILD_HTTP "true") + ELSE () + SET(BUILD_HTTP "false") ENDIF () + ELSE () + SET(BUILD_HTTP "true") ENDIF () -ELSE () +ENDIF () +MESSAGE("after BUILD_HTTP: " ${BUILD_HTTP}) + +IF (${BUILD_HTTP} MATCHES "true") SET(TD_BUILD_HTTP TRUE) +ENDIF () + +IF (TD_BUILD_HTTP) ADD_DEFINITIONS(-DHTTP_EMBEDDED) ENDIF () diff --git a/cmake/input.inc b/cmake/input.inc index a6eaaa97898bbba5b4ba79fac35b0d96c6a9391f..5bd1a7bed6fe9b0c7dc51c46870d8109462eae81 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -92,10 +92,6 @@ ENDIF () SET(TD_BUILD_HTTP FALSE) -IF (${BUILD_HTTP} MATCHES "true") - SET(TD_BUILD_HTTP TRUE) -ENDIF () - SET(TD_MEMORY_SANITIZER FALSE) IF (${MEMORY_SANITIZER} MATCHES "true") SET(TD_MEMORY_SANITIZER TRUE) diff --git a/deps/lz4/inc/lz4.h b/deps/lz4/inc/lz4.h index 43ccb22c9cdb7006b7dab515613580ae4fb8b7a4..7ab1e483a9f53798f6160bd8aaaabd3852e2c146 100644 --- a/deps/lz4/inc/lz4.h +++ b/deps/lz4/inc/lz4.h @@ -1,7 +1,7 @@ /* * LZ4 - Fast LZ compression algorithm * Header File - * Copyright (C) 2011-2017, Yann Collet. + * Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -46,24 +46,31 @@ extern "C" { /** Introduction - LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, + LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core, scalable with multi-cores CPU. It features an extremely fast decoder, with speed in multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. The LZ4 compression library provides in-memory compression and decompression functions. + It gives full buffer control to user. Compression can be done in: - a single step (described as Simple Functions) - a single step, reusing a context (described in Advanced Functions) - unbounded multiple steps (described as Streaming compression) - lz4.h provides block compression functions. It gives full buffer control to user. - Decompressing an lz4-compressed block also requires metadata (such as compressed size). - Each application is free to encode such metadata in whichever way it wants. + lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md). + Decompressing such a compressed block requires additional metadata. + Exact metadata depends on exact decompression function. + For the typical case of LZ4_decompress_safe(), + metadata includes block's compressed size, and maximum bound of decompressed size. + Each application is free to encode and pass such metadata in whichever way it wants. - An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), - take care of encoding standard metadata alongside LZ4-compressed blocks. - If your application requires interoperability, it's recommended to use it. - A library is provided to take care of it, see lz4frame.h. + lz4.h only handle blocks, it can not generate Frames. + + Blocks are different from Frames (doc/lz4_Frame_format.md). + Frames bundle both blocks and metadata in a specified manner. + Embedding metadata is required for compressed data to be self-contained and portable. + Frame format is delivered through a companion API, declared in lz4frame.h. + The `lz4` CLI can only manage frames. */ /*^*************************************************************** @@ -72,27 +79,28 @@ extern "C" { /* * LZ4_DLL_EXPORT : * Enable exporting of functions when building a Windows DLL -* LZ4LIB_API : +* LZ4LIB_VISIBILITY : * Control library symbols visibility. */ - -#include - +#ifndef LZ4LIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define LZ4LIB_VISIBILITY +# endif +#endif #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) -# define LZ4LIB_API __declspec(dllexport) +# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) -# define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ -#elif defined(__GNUC__) && (__GNUC__ >= 4) -# define LZ4LIB_API __attribute__ ((__visibility__ ("default"))) +# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define LZ4LIB_API +# define LZ4LIB_API LZ4LIB_VISIBILITY #endif - /*------ Version ------*/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) @@ -101,8 +109,8 @@ extern "C" { #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) -LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */ -LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */ +LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */ +LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */ /*-************************************ @@ -111,42 +119,49 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; /*! * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) - * Increasing memory usage improves compression ratio - * Reduced memory usage can improve speed, due to cache effect + * Increasing memory usage improves compression ratio. + * Reduced memory usage may improve speed, thanks to better cache locality. * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #ifndef LZ4_MEMORY_USAGE # define LZ4_MEMORY_USAGE 14 #endif + /*-************************************ * Simple Functions **************************************/ /*! LZ4_compress_default() : - Compresses 'sourceSize' bytes from buffer 'source' - into already allocated 'dest' buffer of size 'maxDestSize'. - Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'source' into a more limited 'dest' budget, - compression stops *immediately*, and the function result is zero. - As a consequence, 'dest' content is not valid. - This function never writes outside 'dest' buffer, nor read outside 'source' buffer. - sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE - maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) - return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); + * Compresses 'srcSize' bytes from buffer 'src' + * into already allocated 'dst' buffer of size 'dstCapacity'. + * Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). + * It also runs faster, so it's a recommended setting. + * If the function cannot compress 'src' into a more limited 'dst' budget, + * compression stops *immediately*, and the function result is zero. + * In which case, 'dst' content is undefined (invalid). + * srcSize : max supported value is LZ4_MAX_INPUT_SIZE. + * dstCapacity : size of buffer 'dst' (which must be already allocated) + * @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) + * or 0 if compression fails + * Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer). + */ +LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_safe() : - compressedSize : is the precise full size of the compressed block. - maxDecompressedSize : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) - If destination buffer is not large enough, decoding will stop and output an error code (<0). - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function is protected against buffer overflow exploits, including malicious data packets. - It never writes outside output buffer, nor reads outside input buffer. -*/ -LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + * compressedSize : is the exact complete size of the compressed block. + * dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size. + * @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) + * If destination buffer is not large enough, decoding will stop and output an error code (negative value). + * If the source stream is detected malformed, the function will stop decoding and return a negative result. + * Note 1 : This function is protected against malicious data packets : + * it will never writes outside 'dst' buffer, nor read outside 'source' buffer, + * even if the compressed block is maliciously modified to order the decoder to do these actions. + * In such case, the decoder stops immediately, and considers the compressed block malformed. + * Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them. + * The implementation is free to send / store / derive this information in whichever way is most beneficial. + * If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead. + */ +LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); /*-************************************ @@ -155,322 +170,603 @@ LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compress #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) -/*! -LZ4_compressBound() : +/*! LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). - Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize) inputSize : max supported value is LZ4_MAX_INPUT_SIZE return : maximum output size in a "worst case" scenario - or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) + or 0, if input size is incorrect (too large or negative) */ LZ4LIB_API int LZ4_compressBound(int inputSize); -/*! -LZ4_compress_fast() : - Same as LZ4_compress_default(), but allows to select an "acceleration" factor. +/*! LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows selection of "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() - Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. + Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT (currently == 1, see lz4.c). + Values > LZ4_ACCELERATION_MAX will be replaced by LZ4_ACCELERATION_MAX (currently == 65537, see lz4.c). */ -LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/*! -LZ4_compress_fast_extState() : - Same compression function, just using an externally allocated memory space to store compression state. - Use LZ4_sizeofState() to know how much memory must be allocated, - and allocate it on 8-bytes boundaries (using malloc() typically). - Then, provide it as 'void* state' to compression function. -*/ +/*! LZ4_compress_fast_extState() : + * Same as LZ4_compress_fast(), using an externally allocated memory space for its state. + * Use LZ4_sizeofState() to know how much memory must be allocated, + * and allocate it on 8-bytes boundaries (using `malloc()` typically). + * Then, provide this buffer as `void* state` to compression function. + */ LZ4LIB_API int LZ4_sizeofState(void); -LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); +LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); -/*! -LZ4_compress_destSize() : - Reverse the logic, by compressing as much data as possible from 'source' buffer - into already allocated buffer 'dest' of size 'targetDestSize'. - This function either compresses the entire 'source' content into 'dest' if it's large enough, - or fill 'dest' buffer completely with as much data as possible from 'source'. - *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. - New value is necessarily <= old value. - return : Nb bytes written into 'dest' (necessarily <= targetDestSize) - or 0 if compression fails -*/ -LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); - +/*! LZ4_compress_destSize() : + * Reverse the logic : compresses as much data as possible from 'src' buffer + * into already allocated buffer 'dst', of size >= 'targetDestSize'. + * This function either compresses the entire 'src' content into 'dst' if it's large enough, + * or fill 'dst' buffer completely with as much data as possible from 'src'. + * note: acceleration parameter is fixed to "default". + * + * *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. + * New value is necessarily <= input value. + * @return : Nb bytes written into 'dst' (necessarily <= targetDestSize) + * or 0 if compression fails. + * + * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+): + * the produced compressed content could, in specific circumstances, + * require to be decompressed into a destination buffer larger + * by at least 1 byte than the content to decompress. + * If an application uses `LZ4_compress_destSize()`, + * it's highly recommended to update liblz4 to v1.9.2 or better. + * If this can't be done or ensured, + * the receiving decompression function should provide + * a dstCapacity which is > decompressedSize, by at least 1 byte. + * See https://github.com/lz4/lz4/issues/859 for details + */ +LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); -/*! -LZ4_decompress_fast() : - originalSize : is the original and therefore uncompressed size - return : the number of bytes read from the source buffer (in other words, the compressed size) - If the source stream is detected malformed, the function will stop decoding and return a negative result. - Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. - note : This function fully respect memory boundaries for properly formed compressed data. - It is a bit faster than LZ4_decompress_safe(). - However, it does not provide any protection against intentionally modified data stream (malicious input). - Use this function in trusted environment only (data to decode comes from a trusted source). -*/ -LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); -/*! -LZ4_decompress_safe_partial() : - This function decompress a compressed block of size 'compressedSize' at position 'source' - into destination buffer 'dest' of size 'maxDecompressedSize'. - The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, - reducing decompression time. - return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) - Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. - Always control how many bytes were decoded. - If the source stream is detected malformed, the function will stop decoding and return a negative result. - This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets -*/ -LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); +/*! LZ4_decompress_safe_partial() : + * Decompress an LZ4 compressed block, of size 'srcSize' at position 'src', + * into destination buffer 'dst' of size 'dstCapacity'. + * Up to 'targetOutputSize' bytes will be decoded. + * The function stops decoding on reaching this objective. + * This can be useful to boost performance + * whenever only the beginning of a block is required. + * + * @return : the number of bytes decoded in `dst` (necessarily <= targetOutputSize) + * If source stream is detected malformed, function returns a negative result. + * + * Note 1 : @return can be < targetOutputSize, if compressed block contains less data. + * + * Note 2 : targetOutputSize must be <= dstCapacity + * + * Note 3 : this function effectively stops decoding on reaching targetOutputSize, + * so dstCapacity is kind of redundant. + * This is because in older versions of this function, + * decoding operation would still write complete sequences. + * Therefore, there was no guarantee that it would stop writing at exactly targetOutputSize, + * it could write more bytes, though only up to dstCapacity. + * Some "margin" used to be required for this operation to work properly. + * Thankfully, this is no longer necessary. + * The function nonetheless keeps the same signature, in an effort to preserve API compatibility. + * + * Note 4 : If srcSize is the exact size of the block, + * then targetOutputSize can be any value, + * including larger than the block's decompressed size. + * The function will, at most, generate block's decompressed size. + * + * Note 5 : If srcSize is _larger_ than block's compressed size, + * then targetOutputSize **MUST** be <= block's decompressed size. + * Otherwise, *silent corruption will occur*. + */ +LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); /*-********************************************* * Streaming Compression Functions ***********************************************/ -typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ +typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ -/*! LZ4_createStream() and LZ4_freeStream() : - * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. - * LZ4_freeStream() releases its memory. - */ LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); -/*! LZ4_resetStream() : - * An LZ4_stream_t structure can be allocated once and re-used multiple times. - * Use this function to start compressing a new stream. +/*! LZ4_resetStream_fast() : v1.9.0+ + * Use this to prepare an LZ4_stream_t for a new chain of dependent blocks + * (e.g., LZ4_compress_fast_continue()). + * + * An LZ4_stream_t must be initialized once before usage. + * This is automatically done when created by LZ4_createStream(). + * However, should the LZ4_stream_t be simply declared on stack (for example), + * it's necessary to initialize it first, using LZ4_initStream(). + * + * After init, start any new stream with LZ4_resetStream_fast(). + * A same LZ4_stream_t can be re-used multiple times consecutively + * and compress multiple streams, + * provided that it starts each new stream with LZ4_resetStream_fast(). + * + * LZ4_resetStream_fast() is much faster than LZ4_initStream(), + * but is not compatible with memory regions containing garbage data. + * + * Note: it's only useful to call LZ4_resetStream_fast() + * in the context of streaming compression. + * The *extState* functions perform their own resets. + * Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive. */ -LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); +LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr); /*! LZ4_loadDict() : - * Use this function to load a static dictionary into LZ4_stream_t. - * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Use this function to reference a static dictionary into LZ4_stream_t. + * The dictionary must remain available during compression. + * LZ4_loadDict() triggers a reset, so any previous data will be forgotten. + * The same dictionary will have to be loaded on decompression side for successful decoding. + * Dictionary are useful for better compression of small data (KB range). + * While LZ4 accept any input as dictionary, + * results are generally better when using Zstandard's Dictionary Builder. * Loading a size of 0 is allowed, and is the same as reset. - * @return : dictionary size, in bytes (necessarily <= 64 KB) + * @return : loaded dictionary size, in bytes (necessarily <= 64 KB) */ LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /*! LZ4_compress_fast_continue() : - * Compress content into 'src' using data from previously compressed blocks, improving compression ratio. - * 'dst' buffer must be already allocated. + * Compress 'src' content using data from previously compressed blocks, for better compression ratio. + * 'dst' buffer must be already allocated. * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * - * Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! - * Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. - * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. - * * @return : size of compressed block - * or 0 if there is an error (typically, compressed data cannot fit into 'dst') - * After an error, the stream status is invalid, it can only be reset or freed. + * or 0 if there is an error (typically, cannot fit into 'dst'). + * + * Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block. + * Each block has precise boundaries. + * Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata. + * It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together. + * + * Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory ! + * + * Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB. + * Make sure that buffers are separated, by at least one byte. + * This construction ensures that each block only depends on previous block. + * + * Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. + * + * Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed. */ LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); /*! LZ4_saveDict() : - * If previously compressed data block is not guaranteed to remain available at its current memory location, + * If last 64KB data cannot be guaranteed to remain available at its current memory location, * save it into a safer place (char* safeBuffer). - * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. - * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. + * This is schematically equivalent to a memcpy() followed by LZ4_loadDict(), + * but is much faster, because LZ4_saveDict() doesn't need to rebuild tables. + * @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error. */ -LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); +LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize); /*-********************************************** * Streaming Decompression Functions * Bufferless synchronous API ************************************************/ -typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ +typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */ /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : - * creation / destruction of streaming decompression tracking structure. - * A tracking structure can be re-used multiple times sequentially. */ + * creation / destruction of streaming decompression tracking context. + * A tracking context can be re-used multiple times. + */ LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); /*! LZ4_setStreamDecode() : - * An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. + * An LZ4_streamDecode_t context can be allocated once and re-used multiple times. * Use this function to start decompression of a new stream of blocks. - * A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. + * A dictionary can optionally be set. Use NULL or size 0 for a reset order. + * Dictionary is presumed stable : it must remain accessible and unmodified during next decompression. * @return : 1 if OK, 0 if error */ LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); +/*! LZ4_decoderRingBufferSize() : v1.8.2+ + * Note : in a ring buffer scenario (optional), + * blocks are presumed decompressed next to each other + * up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize), + * at which stage it resumes from beginning of ring buffer. + * When setting such a ring buffer for streaming decompression, + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize); +#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */ + /*! LZ4_decompress_*_continue() : * These decoding functions allow decompression of consecutive blocks in "streaming" mode. * A block is an unsplittable entity, it must be presented entirely to a decompression function. - * Decompression functions only accept one block at a time. - * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). - * - * Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : - * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) - * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. - * maxBlockSize is implementation dependent. It's the maximum size of any single block. + * Decompression functions only accepts one block at a time. + * The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded. + * If less than 64KB of data has been decoded, all the data must be present. + * + * Special : if decompression side sets a ring buffer, it must respect one of the following conditions : + * - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize). + * maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes. + * In which case, encoding and decoding buffers do not need to be synchronized. + * Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize. + * - Synchronized mode : + * Decompression buffer size is _exactly_ the same as compression buffer size, + * and follows exactly same update rule (block boundaries at same positions), + * and decoding function is provided with exact decompressed size of each block (exception for last block of the stream), + * _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB). + * - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes. * In which case, encoding and decoding buffers do not need to be synchronized, * and encoding ring buffer can have any size, including small ones ( < 64 KB). - * - _At least_ 64 KB + 8 bytes + maxBlockSize. - * In which case, encoding and decoding buffers do not need to be synchronized, - * and encoding ring buffer can have any size, including larger than decoding buffer. - * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, - * and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. + * + * Whenever these conditions are not possible, + * save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression, + * then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block. */ -LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); -LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); +LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); /*! LZ4_decompress_*_usingDict() : * These decoding functions work the same as * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() * They are stand-alone, and don't need an LZ4_streamDecode_t structure. + * Dictionary is presumed stable : it must remain accessible and unmodified during decompression. + * Performance tip : Decompression speed can be substantially increased + * when dst == dictStart + dictSize. */ -LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); -LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); +LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); + +#endif /* LZ4_H_2983827168210 */ -/*^********************************************** +/*^************************************* * !!!!!! STATIC LINKING ONLY !!!!!! - ***********************************************/ -/*-************************************ - * Private definitions - ************************************** - * Do not use these definitions. - * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. - * Using these definitions will expose code to API and/or ABI break in future versions of the library. - **************************************/ + ***************************************/ + +/*-**************************************************************************** + * Experimental section + * + * Symbols declared in this section must be considered unstable. Their + * signatures or semantics may change, or they may be removed altogether in the + * future. They are therefore only safe to depend on when the caller is + * statically linked against the library. + * + * To protect against unsafe usage, not only are the declarations guarded, + * the definitions are hidden by default + * when building LZ4 as a shared/dynamic library. + * + * In order to access these declarations, + * define LZ4_STATIC_LINKING_ONLY in your application + * before including LZ4's headers. + * + * In order to make their implementations accessible dynamically, you must + * define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library. + ******************************************************************************/ + +#ifdef LZ4_STATIC_LINKING_ONLY + +#ifndef LZ4_STATIC_3504398509 +#define LZ4_STATIC_3504398509 + +#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS +#define LZ4LIB_STATIC_API LZ4LIB_API +#else +#define LZ4LIB_STATIC_API +#endif + + +/*! LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. + * It is only safe to call if the state buffer is known to be correctly initialized already + * (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized"). + * From a high level, the difference is that + * this function initializes the provided state with a call to something like LZ4_resetStream_fast() + * while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream(). + */ +LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); + +/*! LZ4_attach_dictionary() : + * This is an experimental API that allows + * efficient use of a static dictionary many times. + * + * Rather than re-loading the dictionary buffer into a working context before + * each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a + * working LZ4_stream_t, this function introduces a no-copy setup mechanism, + * in which the working stream references the dictionary stream in-place. + * + * Several assumptions are made about the state of the dictionary stream. + * Currently, only streams which have been prepared by LZ4_loadDict() should + * be expected to work. + * + * Alternatively, the provided dictionaryStream may be NULL, + * in which case any existing dictionary stream is unset. + * + * If a dictionary is provided, it replaces any pre-existing stream history. + * The dictionary contents are the only history that can be referenced and + * logically immediately precede the data compressed in the first subsequent + * compression call. + * + * The dictionary will only remain attached to the working stream through the + * first compression call, at the end of which it is cleared. The dictionary + * stream (and source buffer) must remain in-place / accessible / unchanged + * through the completion of the first compression call on the stream. + */ +LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream); + + +/*! In-place compression and decompression + * + * It's possible to have input and output sharing the same buffer, + * for highly contrained memory environments. + * In both cases, it requires input to lay at the end of the buffer, + * and decompression to start at beginning of the buffer. + * Buffer size must feature some margin, hence be larger than final size. + * + * |<------------------------buffer--------------------------------->| + * |<-----------compressed data--------->| + * |<-----------decompressed size------------------>| + * |<----margin---->| + * + * This technique is more useful for decompression, + * since decompressed size is typically larger, + * and margin is short. + * + * In-place decompression will work inside any buffer + * which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize). + * This presumes that decompressedSize > compressedSize. + * Otherwise, it means compression actually expanded data, + * and it would be more efficient to store such data with a flag indicating it's not compressed. + * This can happen when data is not compressible (already compressed, or encrypted). + * + * For in-place compression, margin is larger, as it must be able to cope with both + * history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX, + * and data expansion, which can happen when input is not compressible. + * As a consequence, buffer size requirements are much higher, + * and memory savings offered by in-place compression are more limited. + * + * There are ways to limit this cost for compression : + * - Reduce history size, by modifying LZ4_DISTANCE_MAX. + * Note that it is a compile-time constant, so all compressions will apply this limit. + * Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX, + * so it's a reasonable trick when inputs are known to be small. + * - Require the compressor to deliver a "maximum compressed size". + * This is the `dstCapacity` parameter in `LZ4_compress*()`. + * When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail, + * in which case, the return code will be 0 (zero). + * The caller must be ready for these cases to happen, + * and typically design a backup scheme to send data uncompressed. + * The combination of both techniques can significantly reduce + * the amount of margin required for in-place compression. + * + * In-place compression can work in any buffer + * which size is >= (maxCompressedSize) + * with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success. + * LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX, + * so it's possible to reduce memory requirements by playing with them. + */ + +#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32) +#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */ + +#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */ +# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ +#endif + +#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */ +#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */ + +#endif /* LZ4_STATIC_3504398509 */ +#endif /* LZ4_STATIC_LINKING_ONLY */ + + + +#ifndef LZ4_H_98237428734687 +#define LZ4_H_98237428734687 + +/*-************************************************************ + * Private Definitions + ************************************************************** + * Do not use these definitions directly. + * They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. + * Accessing members will expose user code to API and/or ABI break in future versions of the library. + **************************************************************/ #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -#include - -typedef struct { - uint32_t hashTable[LZ4_HASH_SIZE_U32]; - uint32_t currentOffset; - uint32_t initCheck; - const uint8_t* dictionary; - uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ - uint32_t dictSize; -} LZ4_stream_t_internal; - -typedef struct { - const uint8_t* externalDict; - size_t extDictSize; - const uint8_t* prefixEnd; - size_t prefixSize; -} LZ4_streamDecode_t_internal; - +# include + typedef int8_t LZ4_i8; + typedef uint8_t LZ4_byte; + typedef uint16_t LZ4_u16; + typedef uint32_t LZ4_u32; #else + typedef signed char LZ4_i8; + typedef unsigned char LZ4_byte; + typedef unsigned short LZ4_u16; + typedef unsigned int LZ4_u32; +#endif -typedef struct { - unsigned int hashTable[LZ4_HASH_SIZE_U32]; - unsigned int currentOffset; - unsigned int initCheck; - const unsigned char* dictionary; - unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ - unsigned int dictSize; -} LZ4_stream_t_internal; +typedef struct LZ4_stream_t_internal LZ4_stream_t_internal; +struct LZ4_stream_t_internal { + LZ4_u32 hashTable[LZ4_HASH_SIZE_U32]; + LZ4_u32 currentOffset; + LZ4_u32 tableType; + const LZ4_byte* dictionary; + const LZ4_stream_t_internal* dictCtx; + LZ4_u32 dictSize; +}; typedef struct { - const unsigned char* externalDict; + const LZ4_byte* externalDict; size_t extDictSize; - const unsigned char* prefixEnd; + const LZ4_byte* prefixEnd; size_t prefixSize; } LZ4_streamDecode_t_internal; -#endif -/*! - * LZ4_stream_t : - * information structure to track an LZ4 stream. - * init this structure before first use. - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * it may change in a future version ! +/*! LZ4_stream_t : + * Do not use below internal definitions directly ! + * Declare or allocate an LZ4_stream_t instead. + * LZ4_stream_t can also be created using LZ4_createStream(), which is recommended. + * The structure definition can be convenient for static allocation + * (on stack, or as part of larger structure). + * Init this structure with LZ4_initStream() before first use. + * note : only use this definition in association with static linking ! + * this definition is not API/ABI safe, and may change in future versions. */ -#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(uint64_t)) +#define LZ4_STREAMSIZE 16416 /* static size, for inter-version compatibility */ +#define LZ4_STREAMSIZE_VOIDP (LZ4_STREAMSIZE / sizeof(void*)) union LZ4_stream_u { - uint64_t table[LZ4_STREAMSIZE_U64]; + void* table[LZ4_STREAMSIZE_VOIDP]; LZ4_stream_t_internal internal_donotuse; -} ; /* previously typedef'd to LZ4_stream_t */ +}; /* previously typedef'd to LZ4_stream_t */ -/*! - * LZ4_streamDecode_t : - * information structure to track an LZ4 stream during decompression. - * init this structure using LZ4_setStreamDecode (or memset()) before first use - * note : only use in association with static linking ! - * this definition is not API/ABI safe, - * and may change in a future version ! +/*! LZ4_initStream() : v1.9.0+ + * An LZ4_stream_t structure must be initialized at least once. + * This is automatically done when invoking LZ4_createStream(), + * but it's not when the structure is simply declared on stack (for example). + * + * Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t. + * It can also initialize any arbitrary buffer of sufficient size, + * and will @return a pointer of proper type upon initialization. + * + * Note : initialization fails if size and alignment conditions are not respected. + * In which case, the function will @return NULL. + * Note2: An LZ4_stream_t structure guarantees correct alignment and size. + * Note3: Before v1.9.0, use LZ4_resetStream() instead + */ +LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size); + + +/*! LZ4_streamDecode_t : + * information structure to track an LZ4 stream during decompression. + * init this structure using LZ4_setStreamDecode() before first use. + * note : only use in association with static linking ! + * this definition is not API/ABI safe, + * and may change in a future version ! */ -#define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(uint64_t)) +#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ ) +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { - uint64_t table[LZ4_STREAMDECODESIZE_U64]; + unsigned long long table[LZ4_STREAMDECODESIZE_U64]; LZ4_streamDecode_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_streamDecode_t */ + /*-************************************ * Obsolete Functions **************************************/ /*! Deprecation warnings - Should deprecation warnings be a problem, - it is generally possible to disable them, - typically with -Wno-deprecated-declarations for gcc - or _CRT_SECURE_NO_WARNINGS in Visual. - Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ + * + * Deprecated functions make the compiler generate a warning when invoked. + * This is meant to invite users to update their source code. + * Should deprecation warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc + * or _CRT_SECURE_NO_WARNINGS in Visual. + * + * Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS + * before including the header file. + */ #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ #else -# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */ -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ # define LZ4_DEPRECATED(message) [[deprecated(message)]] -# elif (LZ4_GCC_VERSION >= 405) -# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) -# elif (LZ4_GCC_VERSION >= 301) -# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 45)) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ * 10 + __GNUC_MINOR__ >= 31) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) # else -# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") -# define LZ4_DEPRECATED(message) +# pragma message("WARNING: LZ4_DEPRECATED needs custom implementation for this compiler") +# define LZ4_DEPRECATED(message) /* disabled */ # endif #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ -/* Obsolete compression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +/*! Obsolete compression functions (since v1.7.3) */ +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize); +LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); -/* Obsolete decompression functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); +/*! Obsolete decompression functions (since v1.8.0) */ +LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize); +LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); + +/* Obsolete streaming functions (since v1.7.0) + * degraded functionality; do not use! + * + * In order to perform streaming compression, these functions depended on data + * that is no longer tracked in the state. They have been preserved as well as + * possible: using them will still produce a correct output. However, they don't + * actually retain any history between compression calls. The compression ratio + * achieved will therefore be no better than compressing each chunk + * independently. + */ +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state); + +/*! Obsolete streaming decoding functions (since v1.7.0) */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + +/*! Obsolete LZ4_decompress_fast variants (since v1.9.0) : + * These functions used to be faster than LZ4_decompress_safe(), + * but this is no longer the case. They are now slower. + * This is because LZ4_decompress_fast() doesn't know the input size, + * and therefore must progress more cautiously into the input buffer to not read beyond the end of block. + * On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability. + * As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated. + * + * The last remaining LZ4_decompress_fast() specificity is that + * it can decompress a block without knowing its compressed size. + * Such functionality can be achieved in a more secure manner + * by employing LZ4_decompress_safe_partial(). + * + * Parameters: + * originalSize : is the uncompressed size to regenerate. + * `dst` must be already allocated, its size must be >= 'originalSize' bytes. + * @return : number of bytes read from source buffer (== compressed size). + * The function expects to finish at block's end exactly. + * If the source stream is detected malformed, the function stops decoding and returns a negative result. + * note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer. + * However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds. + * Also, since match offsets are not validated, match reads from 'src' may underflow too. + * These issues never happen if input (compressed) data is correct. + * But they may happen if input data is invalid (error or intentional tampering). + * As a consequence, use these functions in trusted environments with trusted data **only**. + */ +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead") +LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead") +LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); +LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead") +LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); -/* Obsolete streaming functions; use new streaming interface whenever possible */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); +/*! LZ4_resetStream() : + * An LZ4_stream_t structure must be initialized at least once. + * This is done with LZ4_initStream(), or LZ4_resetStream(). + * Consider switching to LZ4_initStream(), + * invoking LZ4_resetStream() will trigger deprecation warnings in the future. + */ +LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); -/* Obsolete streaming decoding functions */ -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); -LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); -#endif /* LZ4_H_2983827168210 */ +#endif /* LZ4_H_98237428734687 */ #if defined (__cplusplus) diff --git a/deps/lz4/src/lz4.c b/deps/lz4/src/lz4.c index c48baa63fc1fd7f666e5e8f183e682e262bb0448..9f5e9bfa0839f8e1347d2abb3d867b21ff740215 100644 --- a/deps/lz4/src/lz4.c +++ b/deps/lz4/src/lz4.c @@ -1,6 +1,6 @@ /* LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2017, Yann Collet. + Copyright (C) 2011-present, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -32,7 +32,6 @@ - LZ4 source repository : https://github.com/lz4/lz4 */ - /*-************************************ * Tuning parameters **************************************/ @@ -46,10 +45,16 @@ #endif /* - * ACCELERATION_DEFAULT : + * LZ4_ACCELERATION_DEFAULT : * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 */ -#define ACCELERATION_DEFAULT 1 +#define LZ4_ACCELERATION_DEFAULT 1 +/* + * LZ4_ACCELERATION_MAX : + * Any "acceleration" value higher than this threshold + * get treated as LZ4_ACCELERATION_MAX instead (fix #876) + */ +#define LZ4_ACCELERATION_MAX 65537 /*-************************************ @@ -69,9 +74,11 @@ * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# if defined(__GNUC__) && \ + ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \ + || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define LZ4_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) # define LZ4_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -80,14 +87,33 @@ * LZ4_FORCE_SW_BITCOUNT * Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for WinCE doesn't support Hardware bit count */ +# undef LZ4_FORCE_SW_BITCOUNT /* avoid double def */ # define LZ4_FORCE_SW_BITCOUNT #endif + /*-************************************ * Dependency **************************************/ +/* + * LZ4_SRC_INCLUDED: + * Amalgamation flag, whether lz4.c is included + */ +#ifndef LZ4_SRC_INCLUDED +# define LZ4_SRC_INCLUDED 1 +#endif + +#ifndef LZ4_STATIC_LINKING_ONLY +#define LZ4_STATIC_LINKING_ONLY +#endif + +#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS +#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */ +#endif + +#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */ #include "lz4.h" /* see also "memory routines" below */ @@ -95,10 +121,9 @@ /*-************************************ * Compiler Options **************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# include -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) /* Visual Studio 2005+ */ +# include /* only present in VS2005+ */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ #endif /* _MSC_VER */ #ifndef LZ4_FORCE_INLINE @@ -117,29 +142,135 @@ # endif /* _MSC_VER */ #endif /* LZ4_FORCE_INLINE */ +/* LZ4_FORCE_O2 and LZ4_FORCE_INLINE + * gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy8, + * together with a simple 8-byte copy loop as a fall-back path. + * However, this optimization hurts the decompression speed by >30%, + * because the execution does not go to the optimized loop + * for typical compressible data, and all of the preamble checks + * before going to the fall-back path become useless overhead. + * This optimization happens only with the -O3 flag, and -O2 generates + * a simple 8-byte copy loop. + * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy8 + * functions are annotated with __attribute__((optimize("O2"))), + * and also LZ4_wildCopy8 is forcibly inlined, so that the O2 attribute + * of LZ4_wildCopy8 does not affect the compression speed. + */ +#if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) && !defined(__clang__) +# define LZ4_FORCE_O2 __attribute__((optimize("O2"))) +# undef LZ4_FORCE_INLINE +# define LZ4_FORCE_INLINE static __inline __attribute__((optimize("O2"),always_inline)) +#else +# define LZ4_FORCE_O2 +#endif + #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) #endif +#ifndef likely #define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely #define unlikely(expr) expect((expr) != 0, 0) +#endif + +/* Should the alignment test prove unreliable, for some reason, + * it can be disabled by setting LZ4_ALIGN_TEST to 0 */ +#ifndef LZ4_ALIGN_TEST /* can be externally provided */ +# define LZ4_ALIGN_TEST 1 +#endif /*-************************************ * Memory routines **************************************/ -#include /* malloc, calloc, free */ -#define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free +#ifdef LZ4_USER_MEMORY_FUNCTIONS +/* memory management functions can be customized by user project. + * Below functions must exist somewhere in the Project + * and be available at link time */ +void* LZ4_malloc(size_t s); +void* LZ4_calloc(size_t n, size_t s); +void LZ4_free(void* p); +# define ALLOC(s) LZ4_malloc(s) +# define ALLOC_AND_ZERO(s) LZ4_calloc(1,s) +# define FREEMEM(p) LZ4_free(p) +#else +# include /* malloc, calloc, free */ +# define ALLOC(s) malloc(s) +# define ALLOC_AND_ZERO(s) calloc(1,s) +# define FREEMEM(p) free(p) +#endif + #include /* memset, memcpy */ -#define MEM_INIT memset +#define MEM_INIT(p,v,s) memset((p),(v),(s)) + + +/*-************************************ +* Common Constants +**************************************/ +#define MINMATCH 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MFLIMIT 12 /* see ../doc/lz4_Block_format.md#parsing-restrictions */ +#define MATCH_SAFEGUARD_DISTANCE ((2*WILDCOPYLENGTH) - MINMATCH) /* ensure it's possible to write 2 x wildcopyLength without overflowing output buffer */ +#define FASTLOOP_SAFE_DISTANCE 64 +static const int LZ4_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define LZ4_DISTANCE_ABSOLUTE_MAX 65535 +#if (LZ4_DISTANCE_MAX > LZ4_DISTANCE_ABSOLUTE_MAX) /* max supported by LZ4 format */ +# error "LZ4_DISTANCE_MAX is too big : must be <= 65535" +#endif + +#define ML_BITS 4 +#define ML_MASK ((1U<=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use after variable declarations */ + +#if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) +# include + static int g_debuglog_enable = 1; +# define DEBUGLOG(l, ...) { \ + if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + +static int LZ4_isAligned(const void* ptr, size_t alignment) +{ + return ((size_t)ptr & (alignment -1)) == 0; +} /*-************************************ -* Basic Types +* Types **************************************/ +#include #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) # include typedef uint8_t BYTE; @@ -149,11 +280,14 @@ typedef uint64_t U64; typedef uintptr_t uptrval; #else +# if UINT_MAX != 4294967295UL +# error "LZ4 code (when not C++ or C99) assumes that sizeof(int) == 4" +# endif typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; - typedef uint64_t U64; + typedef unsigned long long U64; typedef size_t uptrval; /* generally true, except OpenVMS-64 */ #endif @@ -163,9 +297,31 @@ typedef size_t reg_t; /* 32-bits in x32 mode */ #endif +typedef enum { + notLimited = 0, + limitedOutput = 1, + fillOutput = 2 +} limitedOutput_directive; + + /*-************************************ * Reading and writing into memory **************************************/ + +/** + * LZ4 relies on memcpy with a constant size being inlined. In freestanding + * environments, the compiler can't assume the implementation of memcpy() is + * standard compliant, so it can't apply its specialized memcpy() inlining + * logic. When possible, use __builtin_memcpy() to tell the compiler to analyze + * memcpy() as if it were standard compliant, so it can inline it in freestanding + * environments. This is needed when decompressing the Linux Kernel, for example. + */ +#if defined(__GNUC__) && (__GNUC__ >= 4) +#define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size) +#else +#define LZ4_memcpy(dst, src, size) memcpy(dst, src, size) +#endif + static unsigned LZ4_isLittleEndian(void) { const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ @@ -196,31 +352,31 @@ static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArc static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -#else /* safe and portable access through memcpy() */ +#else /* safe and portable access using memcpy() */ static U16 LZ4_read16(const void* memPtr) { - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; + U16 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static U32 LZ4_read32(const void* memPtr) { - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; + U32 val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static reg_t LZ4_read_ARCH(const void* memPtr) { - reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; + reg_t val; LZ4_memcpy(&val, memPtr, sizeof(val)); return val; } static void LZ4_write16(void* memPtr, U16 value) { - memcpy(memPtr, &value, sizeof(value)); + LZ4_memcpy(memPtr, &value, sizeof(value)); } static void LZ4_write32(void* memPtr, U32 value) { - memcpy(memPtr, &value, sizeof(value)); + LZ4_memcpy(memPtr, &value, sizeof(value)); } #endif /* LZ4_FORCE_MEMORY_ACCESS */ @@ -247,130 +403,216 @@ static void LZ4_writeLE16(void* memPtr, U16 value) } } -static void LZ4_copy8(void* dst, const void* src) -{ - memcpy(dst,src,8); -} - /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ -static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +LZ4_FORCE_INLINE +void LZ4_wildCopy8(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; BYTE* const e = (BYTE*)dstEnd; - do { LZ4_copy8(d,s); d+=8; s+=8; } while (d= 16. */ +LZ4_FORCE_INLINE void +LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* const e = (BYTE*)dstEnd; -/*-************************************ -* Error detection -**************************************/ -#define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + do { LZ4_memcpy(d,s,16); LZ4_memcpy(d+16,s+16,16); d+=32; s+=32; } while (d=2) -# include -# define DEBUGLOG(l, ...) { \ - if (l<=LZ4_DEBUG) { \ - fprintf(stderr, __FILE__ ": "); \ - fprintf(stderr, __VA_ARGS__); \ - fprintf(stderr, " \n"); \ - } } -#else -# define DEBUGLOG(l, ...) {} /* disabled */ +/* LZ4_memcpy_using_offset() presumes : + * - dstEnd >= dstPtr + MINMATCH + * - there is at least 8 bytes available to write after dstEnd */ +LZ4_FORCE_INLINE void +LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset) +{ + BYTE v[8]; + + assert(dstEnd >= dstPtr + MINMATCH); + + switch(offset) { + case 1: + MEM_INIT(v, *srcPtr, 8); + break; + case 2: + LZ4_memcpy(v, srcPtr, 2); + LZ4_memcpy(&v[2], srcPtr, 2); + LZ4_memcpy(&v[4], v, 4); + break; + case 4: + LZ4_memcpy(v, srcPtr, 4); + LZ4_memcpy(&v[4], srcPtr, 4); + break; + default: + LZ4_memcpy_using_offset_base(dstPtr, srcPtr, dstEnd, offset); + return; + } + + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + while (dstPtr < dstEnd) { + LZ4_memcpy(dstPtr, v, 8); + dstPtr += 8; + } +} #endif /*-************************************ * Common functions **************************************/ -static unsigned LZ4_NbCommonBytes (register reg_t val) +static unsigned LZ4_NbCommonBytes (reg_t val) { + assert(val != 0); if (LZ4_isLittleEndian()) { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r = 0; - _BitScanForward64( &r, (U64)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); + if (sizeof(val) == 8) { +# if defined(_MSC_VER) && (_MSC_VER >= 1800) && defined(_M_AMD64) && !defined(LZ4_FORCE_SW_BITCOUNT) + /* x64 CPUS without BMI support interpret `TZCNT` as `REP BSF` */ + return (unsigned)_tzcnt_u64(val) >> 3; +# elif defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64(&r, (U64)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctzll((U64)val) >> 3; # else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(int64_t)val) * 0x0218A392CDABBD3FULL)) >> 58]; + const U64 m = 0x0101010101010101ULL; + val ^= val - 1; + return (unsigned)(((U64)((val & (m - 1)) * m)) >> 56); # endif } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); +# if defined(_MSC_VER) && (_MSC_VER >= 1400) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward(&r, (U32)val); + return (unsigned)r >> 3; +# elif (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_ctz((U32)val) >> 3; # else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; + const U32 m = 0x01010101; + return (unsigned)((((val - 1) ^ val) & (m - 1)) * m) >> 24; # endif } } else /* Big Endian CPU */ { if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(__TINYC__) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clzll((U64)val) >> 3; # else +#if 1 + /* this method is probably faster, + * but adds a 128 bytes lookup table */ + static const unsigned char ctz7_tab[128] = { + 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + }; + U64 const mask = 0x0101010101010101ULL; + U64 const t = (((val >> 8) - mask) | val) & mask; + return ctz7_tab[(t * 0x0080402010080402ULL) >> 57]; +#else + /* this method doesn't consume memory space like the previous one, + * but it contains several branches, + * that may end up slowing execution */ + static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. + Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. + Note that this code path is never triggered in 32-bits mode. */ unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } r += (!val); return r; +#endif # endif } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - uint64_t r = 0; - _BitScanReverse( &r, (uint64_t)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); +# if (defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))) && \ + !defined(LZ4_FORCE_SW_BITCOUNT) + return (unsigned)__builtin_clz((U32)val) >> 3; # else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; + val >>= 8; + val = ((((val + 0x00FFFF00) | 0x00FFFFFF) + val) | + (val + 0x00FF0000)) >> 24; + return (unsigned)val ^ 3; # endif } } } + #define STEPSIZE sizeof(reg_t) -static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +LZ4_FORCE_INLINE +unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { const BYTE* const pStart = pIn; - while (likely(pIn compression ru /*-************************************ * Local Structures and types **************************************/ -typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; -typedef enum { byPtr, byU32, byU16 } tableType_t; - -typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { clearedTable = 0, byPtr, byU32, byU16 } tableType_t; + +/** + * This enum distinguishes several different modes of accessing previous + * content in the stream. + * + * - noDict : There is no preceding content. + * - withPrefix64k : Table entries up to ctx->dictSize before the current blob + * blob being compressed are valid and refer to the preceding + * content (of length ctx->dictSize), which is available + * contiguously preceding in memory the content currently + * being compressed. + * - usingExtDict : Like withPrefix64k, but the preceding content is somewhere + * else in memory, starting at ctx->dictionary with length + * ctx->dictSize. + * - usingDictCtx : Like usingExtDict, but everything concerning the preceding + * content is in a separate context, pointed to by + * ctx->dictCtx. ctx->dictionary, ctx->dictSize, and table + * entries in the current context that refer to positions + * preceding the beginning of the current compression are + * ignored. Instead, ctx->dictCtx->dictionary and ctx->dictCtx + * ->dictSize describe the location and size of the preceding + * content, and matches are found by looking in the ctx + * ->dictCtx->hashTable. + */ +typedef enum { noDict = 0, withPrefix64k, usingExtDict, usingDictCtx } dict_directive; typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; -typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; -typedef enum { full = 0, partial = 1 } earlyEnd_directive; - /*-************************************ * Local Utils @@ -411,13 +672,30 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive; int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } -int LZ4_sizeofState() { return LZ4_STREAMSIZE; } +int LZ4_sizeofState(void) { return LZ4_STREAMSIZE; } +/*-************************************ +* Internal Definitions used in Tests +**************************************/ +#if defined (__cplusplus) +extern "C" { +#endif + +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize); + +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize); + +#if defined (__cplusplus) +} +#endif + /*-****************************** * Compression functions ********************************/ -static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hash4(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); @@ -425,15 +703,16 @@ static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } -static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) +LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType) { - static const U64 prime5bytes = 889523592379ULL; - static const U64 prime8bytes = 11400714785074694791ULL; const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; - if (LZ4_isLittleEndian()) + if (LZ4_isLittleEndian()) { + const U64 prime5bytes = 889523592379ULL; return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); - else + } else { + const U64 prime8bytes = 11400714785074694791ULL; return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + } } LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) @@ -442,10 +721,37 @@ LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tab return LZ4_hash4(LZ4_read32(p), tableType); } -static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: { /* illegal! */ assert(0); return; } + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = NULL; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = 0; return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = 0; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putIndexOnHash(U32 idx, U32 h, void* tableBase, tableType_t const tableType) +{ + switch (tableType) + { + default: /* fallthrough */ + case clearedTable: /* fallthrough */ + case byPtr: { /* illegal! */ assert(0); return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = idx; return; } + case byU16: { U16* hashTable = (U16*) tableBase; assert(idx < 65536); hashTable[h] = (U16)idx; return; } + } +} + +LZ4_FORCE_INLINE void LZ4_putPositionOnHash(const BYTE* p, U32 h, + void* tableBase, tableType_t const tableType, + const BYTE* srcBase) { switch (tableType) { + case clearedTable: { /* illegal! */ assert(0); return; } case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } @@ -458,71 +764,161 @@ LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } -static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +/* LZ4_getIndexOnHash() : + * Index of match position registered in hash table. + * hash position must be calculated by using base+index, or dictBase+index. + * Assumption 1 : only valid if tableType == byU32 or byU16. + * Assumption 2 : h is presumed valid (within limits of hash table) + */ +LZ4_FORCE_INLINE U32 LZ4_getIndexOnHash(U32 h, const void* tableBase, tableType_t tableType) +{ + LZ4_STATIC_ASSERT(LZ4_MEMORY_USAGE > 2); + if (tableType == byU32) { + const U32* const hashTable = (const U32*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-2))); + return hashTable[h]; + } + if (tableType == byU16) { + const U16* const hashTable = (const U16*) tableBase; + assert(h < (1U << (LZ4_MEMORY_USAGE-1))); + return hashTable[h]; + } + assert(0); return 0; /* forbidden case */ +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, const void* tableBase, tableType_t tableType, const BYTE* srcBase) { - if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } - if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } - { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ + if (tableType == byPtr) { const BYTE* const* hashTable = (const BYTE* const*) tableBase; return hashTable[h]; } + if (tableType == byU32) { const U32* const hashTable = (const U32*) tableBase; return hashTable[h] + srcBase; } + { const U16* const hashTable = (const U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } -LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +LZ4_FORCE_INLINE const BYTE* +LZ4_getPosition(const BYTE* p, + const void* tableBase, tableType_t tableType, + const BYTE* srcBase) { U32 const h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } +LZ4_FORCE_INLINE void +LZ4_prepareTable(LZ4_stream_t_internal* const cctx, + const int inputSize, + const tableType_t tableType) { + /* If the table hasn't been used, it's guaranteed to be zeroed out, and is + * therefore safe to use no matter what mode we're in. Otherwise, we figure + * out if it's safe to leave as is or whether it needs to be reset. + */ + if ((tableType_t)cctx->tableType != clearedTable) { + assert(inputSize >= 0); + if ((tableType_t)cctx->tableType != tableType + || ((tableType == byU16) && cctx->currentOffset + (unsigned)inputSize >= 0xFFFFU) + || ((tableType == byU32) && cctx->currentOffset > 1 GB) + || tableType == byPtr + || inputSize >= 4 KB) + { + DEBUGLOG(4, "LZ4_prepareTable: Resetting table in %p", cctx); + MEM_INIT(cctx->hashTable, 0, LZ4_HASHTABLESIZE); + cctx->currentOffset = 0; + cctx->tableType = (U32)clearedTable; + } else { + DEBUGLOG(4, "LZ4_prepareTable: Re-use hash table (no reset)"); + } + } + + /* Adding a gap, so all previous entries are > LZ4_DISTANCE_MAX back, is faster + * than compressing without a gap. However, compressing with + * currentOffset == 0 is faster still, so we preserve that case. + */ + if (cctx->currentOffset != 0 && tableType == byU32) { + DEBUGLOG(5, "LZ4_prepareTable: adding 64KB to currentOffset"); + cctx->currentOffset += 64 KB; + } + + /* Finally, clear history */ + cctx->dictCtx = NULL; + cctx->dictionary = NULL; + cctx->dictSize = 0; +} /** LZ4_compress_generic() : - inlined, to ensure branches are decided at compilation time */ -LZ4_FORCE_INLINE int LZ4_compress_generic( + * inlined, to ensure branches are decided at compilation time. + * Presumed already validated at this stage: + * - source != NULL + * - inputSize > 0 + */ +LZ4_FORCE_INLINE int LZ4_compress_generic_validated( LZ4_stream_t_internal* const cctx, const char* const source, char* const dest, const int inputSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ const int maxOutputSize, - const limitedOutput_directive outputLimited, + const limitedOutput_directive outputDirective, const tableType_t tableType, - const dict_directive dict, + const dict_directive dictDirective, const dictIssue_directive dictIssue, - const U32 acceleration) + const int acceleration) { + int result; const BYTE* ip = (const BYTE*) source; - const BYTE* base; + + U32 const startIndex = cctx->currentOffset; + const BYTE* base = (const BYTE*) source - startIndex; const BYTE* lowLimit; - const BYTE* const lowRefLimit = ip - cctx->dictSize; - const BYTE* const dictionary = cctx->dictionary; - const BYTE* const dictEnd = dictionary + cctx->dictSize; - const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; + + const LZ4_stream_t_internal* dictCtx = (const LZ4_stream_t_internal*) cctx->dictCtx; + const BYTE* const dictionary = + dictDirective == usingDictCtx ? dictCtx->dictionary : cctx->dictionary; + const U32 dictSize = + dictDirective == usingDictCtx ? dictCtx->dictSize : cctx->dictSize; + const U32 dictDelta = (dictDirective == usingDictCtx) ? startIndex - dictCtx->currentOffset : 0; /* make indexes in dictCtx comparable with index in current context */ + + int const maybe_extMem = (dictDirective == usingExtDict) || (dictDirective == usingDictCtx); + U32 const prefixIdxLimit = startIndex - dictSize; /* used when dictDirective == dictSmall */ + const BYTE* const dictEnd = dictionary ? dictionary + dictSize : dictionary; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const mflimitPlusOne = iend - MFLIMIT + 1; const BYTE* const matchlimit = iend - LASTLITERALS; + /* the dictCtx currentOffset is indexed on the start of the dictionary, + * while a dictionary in the current context precedes the currentOffset */ + const BYTE* dictBase = !dictionary ? NULL : (dictDirective == usingDictCtx) ? + dictionary + dictSize - dictCtx->currentOffset : + dictionary + dictSize - startIndex; + BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; + U32 offset = 0; U32 forwardH; - /* Init conditions */ - if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ - switch(dict) - { - case noDict: - default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; - case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; - case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; + DEBUGLOG(5, "LZ4_compress_generic_validated: srcSize=%i, tableType=%u", inputSize, tableType); + assert(ip != NULL); + /* If init conditions are not met, we don't have to mark stream + * as having dirty context, since no action was taken yet */ + if (outputDirective == fillOutput && maxOutputSize < 1) { return 0; } /* Impossible to store anything */ + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) { return 0; } /* Size too large (not within 64K limit) */ + if (tableType==byPtr) assert(dictDirective==noDict); /* only supported use case with byPtr */ + assert(acceleration >= 1); + + lowLimit = (const BYTE*)source - (dictDirective == withPrefix64k ? dictSize : 0); + + /* Update context state */ + if (dictDirective == usingDictCtx) { + /* Subsequent linked blocks can't use the dictionary. */ + /* Instead, they use the block we just compressed. */ + cctx->dictCtx = NULL; + cctx->dictSize = (U32)inputSize; + } else { + cctx->dictSize += (U32)inputSize; } - if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizecurrentOffset += (U32)inputSize; + cctx->tableType = (U32)tableType; + + if (inputSizehashTable, tableType, base); @@ -530,50 +926,112 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - ptrdiff_t refDelta = 0; const BYTE* match; BYTE* token; + const BYTE* filledIp; /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + if (tableType == byPtr) { + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; do { U32 const h = forwardH; ip = forwardIp; forwardIp += step; step = (searchMatchNb++ >> LZ4_skipTrigger); - if (unlikely(forwardIp > mflimit)) goto _last_literals; + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + + } while ( (match+LZ4_DISTANCE_MAX < ip) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + + } else { /* byU32, byU16 */ + + const BYTE* forwardIp = ip; + int step = 1; + int searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + U32 const current = (U32)(forwardIp - base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex <= current); + assert(forwardIp - base < (ptrdiff_t)(2 GB - 1)); + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimitPlusOne)) goto _last_literals; + assert(ip < mflimitPlusOne); + + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + assert(tableType == byU32); + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + matchIndex += dictDelta; /* make dictCtx index comparable with current context */ lowLimit = dictionary; } else { - refDelta = 0; + match = base + matchIndex; lowLimit = (const BYTE*)source; - } } + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + DEBUGLOG(7, "extDict candidate: matchIndex=%5u < startIndex=%5u", matchIndex, startIndex); + assert(startIndex - matchIndex >= MINMATCH); + match = dictBase + matchIndex; + lowLimit = dictionary; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; + } + } else { /* single continuous memory segment */ + match = base + matchIndex; + } forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + + DEBUGLOG(7, "candidate at pos=%u (offset=%u \n", matchIndex, current - matchIndex); + if ((dictIssue == dictSmall) && (matchIndex < prefixIdxLimit)) { continue; } /* match outside of valid area */ + assert(matchIndex < current); + if ( ((tableType != byU16) || (LZ4_DISTANCE_MAX < LZ4_DISTANCE_ABSOLUTE_MAX)) + && (matchIndex+LZ4_DISTANCE_MAX < current)) { + continue; + } /* too far */ + assert((current - matchIndex) <= LZ4_DISTANCE_MAX); /* match now expected within distance */ + + if (LZ4_read32(match) == LZ4_read32(ip)) { + if (maybe_extMem) offset = current - matchIndex; + break; /* match found */ + } - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } while(1); } /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + filledIp = ip; + while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } /* Encode Literals */ { unsigned const litLength = (unsigned)(ip - anchor); token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; + if ((outputDirective == limitedOutput) && /* Check output buffer overflow */ + (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)) ) { + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + if ((outputDirective == fillOutput) && + (unlikely(op + (litLength+240)/255 /* litlen */ + litLength /* literals */ + 2 /* offset */ + 1 /* token */ + MFLIMIT - MINMATCH /* min last literals so last match is <= end - MFLIMIT */ > olimit))) { + op--; + goto _last_literals; + } if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; + int len = (int)(litLength - RUN_MASK); *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; *op++ = (BYTE)len; @@ -581,37 +1039,87 @@ LZ4_FORCE_INLINE int LZ4_compress_generic( else *token = (BYTE)(litLength< olimit)) { + /* the match was too close to the end, rewind and go to last literals */ + op = token; + goto _last_literals; + } + /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; + if (maybe_extMem) { /* static test */ + DEBUGLOG(6, " with offset=%u (ext if > %i)", offset, (int)(ip - (const BYTE*)source)); + assert(offset <= LZ4_DISTANCE_MAX && offset > 0); + LZ4_writeLE16(op, (U16)offset); op+=2; + } else { + DEBUGLOG(6, " with offset=%u (same segment)", (U32)(ip - match)); + assert(ip-match <= LZ4_DISTANCE_MAX); + LZ4_writeLE16(op, (U16)(ip - match)); op+=2; + } /* Encode MatchLength */ { unsigned matchCode; - if ((dict==usingExtDict) && (lowLimit==dictionary)) { - const BYTE* limit; - match += refDelta; - limit = ip + (dictEnd-match); + if ( (dictDirective==usingExtDict || dictDirective==usingDictCtx) + && (lowLimit==dictionary) /* match within extDict */ ) { + const BYTE* limit = ip + (dictEnd-match); + assert(dictEnd > match); if (limit > matchlimit) limit = matchlimit; matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; + ip += (size_t)matchCode + MINMATCH; if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + unsigned const more = LZ4_count(limit, (const BYTE*)source, matchlimit); matchCode += more; ip += more; } + DEBUGLOG(6, " with matchLength=%u starting in extDict", matchCode+MINMATCH); } else { matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; + ip += (size_t)matchCode + MINMATCH; + DEBUGLOG(6, " with matchLength=%u", matchCode+MINMATCH); } - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; + if ((outputDirective) && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode+240)/255 > olimit)) ) { + if (outputDirective == fillOutput) { + /* Match description too long : reduce it */ + U32 newMatchCode = 15 /* in token */ - 1 /* to avoid needing a zero byte */ + ((U32)(olimit - op) - 1 - LASTLITERALS) * 255; + ip -= matchCode - newMatchCode; + assert(newMatchCode < matchCode); + matchCode = newMatchCode; + if (unlikely(ip <= filledIp)) { + /* We have already filled up to filledIp so if ip ends up less than filledIp + * we have positions in the hash table beyond the current position. This is + * a problem if we reuse the hash table. So we have to remove these positions + * from the hash table. + */ + const BYTE* ptr; + DEBUGLOG(5, "Clearing %u positions", (U32)(filledIp - ip)); + for (ptr = ip; ptr <= filledIp; ++ptr) { + U32 const h = LZ4_hashPosition(ptr, tableType); + LZ4_clearHash(h, cctx->hashTable, tableType); + } + } + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } if (matchCode >= ML_MASK) { *token += ML_MASK; matchCode -= ML_MASK; @@ -626,41 +1134,89 @@ _next_match: } else *token += (BYTE)(matchCode); } + /* Ensure we have enough space for the last literals. */ + assert(!(outputDirective == fillOutput && op + 1 + LASTLITERALS > olimit)); anchor = ip; /* Test end of chunk */ - if (ip > mflimit) break; + if (ip >= mflimitPlusOne) break; /* Fill table */ LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } + if (tableType == byPtr) { + + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( (match+LZ4_DISTANCE_MAX >= ip) + && (LZ4_read32(match) == LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + } else { /* byU32, byU16 */ + + U32 const h = LZ4_hashPosition(ip, tableType); + U32 const current = (U32)(ip-base); + U32 matchIndex = LZ4_getIndexOnHash(h, cctx->hashTable, tableType); + assert(matchIndex < current); + if (dictDirective == usingDictCtx) { + if (matchIndex < startIndex) { + /* there was no match, try the dictionary */ + matchIndex = LZ4_getIndexOnHash(h, dictCtx->hashTable, byU32); + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + matchIndex += dictDelta; + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else if (dictDirective==usingExtDict) { + if (matchIndex < startIndex) { + match = dictBase + matchIndex; + lowLimit = dictionary; /* required for match length counter */ + } else { + match = base + matchIndex; + lowLimit = (const BYTE*)source; /* required for match length counter */ + } + } else { /* single memory segment */ + match = base + matchIndex; + } + LZ4_putIndexOnHash(current, h, cctx->hashTable, tableType); + assert(matchIndex < current); + if ( ((dictIssue==dictSmall) ? (matchIndex >= prefixIdxLimit) : 1) + && (((tableType==byU16) && (LZ4_DISTANCE_MAX == LZ4_DISTANCE_ABSOLUTE_MAX)) ? 1 : (matchIndex+LZ4_DISTANCE_MAX >= current)) + && (LZ4_read32(match) == LZ4_read32(ip)) ) { + token=op++; + *token=0; + if (maybe_extMem) offset = current - matchIndex; + DEBUGLOG(6, "seq.start:%i, literals=%u, match.start:%i", + (int)(anchor-(const BYTE*)source), 0, (int)(ip-(const BYTE*)source)); + goto _next_match; + } + } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); + } _last_literals: /* Encode Last Literals */ - { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; + { size_t lastRun = (size_t)(iend - anchor); + if ( (outputDirective) && /* Check output buffer overflow */ + (op + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > olimit)) { + if (outputDirective == fillOutput) { + /* adapt lastRun to fill 'dst' */ + assert(olimit >= op); + lastRun = (size_t)(olimit-op) - 1/*token*/; + lastRun -= (lastRun + 256 - RUN_MASK) / 256; /*additional length tokens*/ + } else { + assert(outputDirective == limitedOutput); + return 0; /* cannot compress within `dst` budget. Stored indexes in hash table are nonetheless fine */ + } + } + DEBUGLOG(6, "Final literal run : %i literals", (int)lastRun); if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; @@ -669,252 +1225,182 @@ _last_literals: } else { *op++ = (BYTE)(lastRun< 0); + DEBUGLOG(5, "LZ4_compress_generic: compressed %i bytes into %i bytes", inputSize, result); + return result; +} + +/** LZ4_compress_generic() : + * inlined, to ensure branches are decided at compilation time; + * takes care of src == (NULL, 0) + * and forward the rest to LZ4_compress_generic_validated */ +LZ4_FORCE_INLINE int LZ4_compress_generic( + LZ4_stream_t_internal* const cctx, + const char* const src, + char* const dst, + const int srcSize, + int *inputConsumed, /* only written when outputDirective == fillOutput */ + const int dstCapacity, + const limitedOutput_directive outputDirective, + const tableType_t tableType, + const dict_directive dictDirective, + const dictIssue_directive dictIssue, + const int acceleration) +{ + DEBUGLOG(5, "LZ4_compress_generic: srcSize=%i, dstCapacity=%i", + srcSize, dstCapacity); + + if ((U32)srcSize > (U32)LZ4_MAX_INPUT_SIZE) { return 0; } /* Unsupported srcSize, too large (or negative) */ + if (srcSize == 0) { /* src == NULL supported if srcSize == 0 */ + if (outputDirective != notLimited && dstCapacity <= 0) return 0; /* no output, can't write anything */ + DEBUGLOG(5, "Generating an empty block"); + assert(outputDirective == notLimited || dstCapacity >= 1); + assert(dst != NULL); + dst[0] = 0; + if (outputDirective == fillOutput) { + assert (inputConsumed != NULL); + *inputConsumed = 0; + } + return 1; + } + assert(src != NULL); + + return LZ4_compress_generic_validated(cctx, src, dst, srcSize, + inputConsumed, /* only written into if outputDirective == fillOutput */ + dstCapacity, outputDirective, + tableType, dictDirective, dictIssue, acceleration); } int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; - LZ4_resetStream((LZ4_stream_t*)state); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; - + LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse; + assert(ctx != NULL); + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; if (maxOutputSize >= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); + } } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) { + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration); + } } } - -int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +/** + * LZ4_compress_fast_extState_fastReset() : + * A variant of LZ4_compress_fast_extState(). + * + * Using this variant avoids an expensive initialization step. It is only safe + * to call if the state buffer is known to be correctly initialized already + * (see comment in lz4.h on LZ4_resetStream_fast() for a definition of + * "correctly initialized"). + */ +int LZ4_compress_fast_extState_fastReset(void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration) { -#if (LZ4_HEAPMODE) - void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ -#else - LZ4_stream_t ctx; - void* const ctxPtr = &ctx; -#endif - - int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); - -#if (LZ4_HEAPMODE) - FREEMEM(ctxPtr); -#endif - return result; -} - - -int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) -{ - return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); -} - - -/* hidden debug function */ -/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ -int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) -{ - LZ4_stream_t ctx; - LZ4_resetStream(&ctx); - - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); -} - - -/*-****************************** -* *_destSize() variant -********************************/ - -static int LZ4_compress_destSize_generic( - LZ4_stream_t_internal* const ctx, - const char* const src, - char* const dst, - int* const srcSizePtr, - const int targetDstSize, - const tableType_t tableType) -{ - const BYTE* ip = (const BYTE*) src; - const BYTE* base = (const BYTE*) src; - const BYTE* lowLimit = (const BYTE*) src; - const BYTE* anchor = ip; - const BYTE* const iend = ip + *srcSizePtr; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = iend - LASTLITERALS; - - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + targetDstSize; - BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; - BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); - BYTE* const oMaxSeq = oMaxLit - 1 /* token */; - - U32 forwardH; - - - /* Init conditions */ - if (targetDstSize < 1) return 0; /* Impossible to store anything */ - if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ - if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (*srcSizePtrhashTable, tableType, base); - ip++; forwardH = LZ4_hashPosition(ip, tableType); - - /* Main Loop */ - for ( ; ; ) { - const BYTE* match; - BYTE* token; - - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = 1 << LZ4_skipTrigger; - - do { - U32 h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); - - if (unlikely(forwardIp > mflimit)) goto _last_literals; - - match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); - - } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match) != LZ4_read32(ip)) ); - } - - /* Catch up */ - while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } - - /* Encode Literal length */ - { unsigned litLength = (unsigned)(ip - anchor); - token = op++; - if (op + ((litLength+240)/255) + litLength > oMaxLit) { - /* Not enough space for a last match */ - op--; - goto _last_literals; - } - if (litLength>=RUN_MASK) { - unsigned len = litLength - RUN_MASK; - *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; + LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)state)->internal_donotuse; + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + if (dstCapacity >= LZ4_compressBound(srcSize)) { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } - else *token = (BYTE)(litLength< LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration); } - -_next_match: - /* Encode Offset */ - LZ4_writeLE16(op, (U16)(ip-match)); op+=2; - - /* Encode MatchLength */ - { size_t matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - - if (op + ((matchLength+240)/255) > oMaxMatch) { - /* Match description too long : reduce it */ - matchLength = (15-1) + (oMaxMatch-op) * 255; - } - ip += MINMATCH + matchLength; - - if (matchLength>=ML_MASK) { - *token += ML_MASK; - matchLength -= ML_MASK; - while (matchLength >= 255) { matchLength-=255; *op++ = 255; } - *op++ = (BYTE)matchLength; + } else { + if (srcSize < LZ4_64Klimit) { + const tableType_t tableType = byU16; + LZ4_prepareTable(ctx, srcSize, tableType); + if (ctx->currentOffset) { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, dictSmall, acceleration); + } else { + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } - else *token += (BYTE)(matchLength); + } else { + const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + LZ4_prepareTable(ctx, srcSize, tableType); + return LZ4_compress_generic(ctx, src, dst, srcSize, NULL, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration); } + } +} - anchor = ip; - - /* Test end of block */ - if (ip > mflimit) break; - if (op > oMaxSeq) break; - - /* Fill table */ - LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); - - /* Test next position */ - match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); - LZ4_putPosition(ip, ctx->hashTable, tableType, base); - if ( (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); - } +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + int result; +#if (LZ4_HEAPMODE) + LZ4_stream_t* ctxPtr = ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctxPtr == NULL) return 0; +#else + LZ4_stream_t ctx; + LZ4_stream_t* const ctxPtr = &ctx; +#endif + result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); -_last_literals: - /* Encode Last Literals */ - { size_t lastRunSize = (size_t)(iend - anchor); - if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { - /* adapt lastRunSize to fill 'dst' */ - lastRunSize = (oend-op) - 1; - lastRunSize -= (lastRunSize+240)/255; - } - ip = anchor + lastRunSize; +#if (LZ4_HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} - if (lastRunSize >= RUN_MASK) { - size_t accumulator = lastRunSize - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { - if (*srcSizePtr < LZ4_64Klimit) - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); - else - return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); - } + if (*srcSizePtr < LZ4_64Klimit) { + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1); + } else { + tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32; + return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1); + } } } int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (LZ4_HEAPMODE) - LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ + if (ctx == NULL) return 0; #else LZ4_stream_t ctxBody; LZ4_stream_t* ctx = &ctxBody; @@ -936,20 +1422,50 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe LZ4_stream_t* LZ4_createStream(void) { - LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_stream_t* const lz4s = (LZ4_stream_t*)ALLOC(sizeof(LZ4_stream_t)); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ - LZ4_resetStream(lz4s); + DEBUGLOG(4, "LZ4_createStream %p", lz4s); + if (lz4s == NULL) return NULL; + LZ4_initStream(lz4s, sizeof(*lz4s)); return lz4s; } +static size_t LZ4_stream_t_alignment(void) +{ +#if LZ4_ALIGN_TEST + typedef struct { char c; LZ4_stream_t t; } t_a; + return sizeof(t_a) - sizeof(LZ4_stream_t); +#else + return 1; /* effectively disabled */ +#endif +} + +LZ4_stream_t* LZ4_initStream (void* buffer, size_t size) +{ + DEBUGLOG(5, "LZ4_initStream"); + if (buffer == NULL) { return NULL; } + if (size < sizeof(LZ4_stream_t)) { return NULL; } + if (!LZ4_isAligned(buffer, LZ4_stream_t_alignment())) return NULL; + MEM_INIT(buffer, 0, sizeof(LZ4_stream_t_internal)); + return (LZ4_stream_t*)buffer; +} + +/* resetStream is now deprecated, + * prefer initStream() which is more general */ void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { - MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); + DEBUGLOG(5, "LZ4_resetStream (ctx:%p)", LZ4_stream); + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t_internal)); +} + +void LZ4_resetStream_fast(LZ4_stream_t* ctx) { + LZ4_prepareTable(&(ctx->internal_donotuse), 0, byU32); } int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { if (!LZ4_stream) return 0; /* support free on NULL */ + DEBUGLOG(5, "LZ4_freeStream %p", LZ4_stream); FREEMEM(LZ4_stream); return (0); } @@ -959,43 +1475,82 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; + const tableType_t tableType = byU32; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; - if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ - LZ4_resetStream(LZ4_dict); + DEBUGLOG(4, "LZ4_loadDict (%i bytes from %p into %p)", dictSize, dictionary, LZ4_dict); + + /* It's necessary to reset the context, + * and not just continue it with prepareTable() + * to avoid any risk of generating overflowing matchIndex + * when compressing using this dictionary */ + LZ4_resetStream(LZ4_dict); + + /* We always increment the offset by 64 KB, since, if the dict is longer, + * we truncate it to the last 64k, and if it's shorter, we still want to + * advance by a whole window length so we can provide the guarantee that + * there are only valid offsets in the window, which allows an optimization + * in LZ4_compress_fast_continue() where it uses noDictIssue even when the + * dictionary isn't a full 64k. */ + dict->currentOffset += 64 KB; if (dictSize < (int)HASH_UNIT) { - dict->dictionary = NULL; - dict->dictSize = 0; return 0; } if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; - dict->currentOffset += 64 KB; - base = p - dict->currentOffset; + base = dictEnd - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); - dict->currentOffset += dict->dictSize; + dict->tableType = (U32)tableType; while (p <= dictEnd-HASH_UNIT) { - LZ4_putPosition(p, dict->hashTable, byU32, base); + LZ4_putPosition(p, dict->hashTable, tableType, base); p+=3; } - return dict->dictSize; + return (int)dict->dictSize; +} + +void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream) { + const LZ4_stream_t_internal* dictCtx = dictionaryStream == NULL ? NULL : + &(dictionaryStream->internal_donotuse); + + DEBUGLOG(4, "LZ4_attach_dictionary (%p, %p, size %u)", + workingStream, dictionaryStream, + dictCtx != NULL ? dictCtx->dictSize : 0); + + if (dictCtx != NULL) { + /* If the current offset is zero, we will never look in the + * external dictionary context, since there is no value a table + * entry can take that indicate a miss. In that case, we need + * to bump the offset to something non-zero. + */ + if (workingStream->internal_donotuse.currentOffset == 0) { + workingStream->internal_donotuse.currentOffset = 64 KB; + } + + /* Don't actually attach an empty dictionary. + */ + if (dictCtx->dictSize == 0) { + dictCtx = NULL; + } + } + workingStream->internal_donotuse.dictCtx = dictCtx; } -static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, int nextSize) { - if ((LZ4_dict->currentOffset > 0x80000000) || - ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ + assert(nextSize >= 0); + if (LZ4_dict->currentOffset + (unsigned)nextSize > 0x80000000) { /* potential ptrdiff_t overflow (32-bits mode) */ /* rescale hash table */ U32 const delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; + DEBUGLOG(4, "LZ4_renormDictT"); for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; @@ -1007,16 +1562,29 @@ static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) } -int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, + const char* source, char* dest, + int inputSize, int maxOutputSize, + int acceleration) { + const tableType_t tableType = byU32; LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + const BYTE* dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = (const BYTE*) source; - if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ - if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; - LZ4_renormDictT(streamPtr, smallest); - if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + DEBUGLOG(5, "LZ4_compress_fast_continue (inputSize=%i)", inputSize); + + LZ4_renormDictT(streamPtr, inputSize); /* avoid index overflow */ + if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT; + if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX; + + /* invalidate tiny dictionaries */ + if ( (streamPtr->dictSize-1 < 4-1) /* intentional underflow */ + && (dictEnd != (const BYTE*)source) ) { + DEBUGLOG(5, "LZ4_compress_fast_continue: dictSize(%u) at addr:%p is too small", streamPtr->dictSize, streamPtr->dictionary); + streamPtr->dictSize = 0; + streamPtr->dictionary = (const BYTE*)source; + dictEnd = (const BYTE*)source; + } /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; @@ -1030,46 +1598,61 @@ int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, ch /* prefix mode : source data follows dictionary */ if (dictEnd == (const BYTE*)source) { - int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, dictSmall, acceleration); else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); - streamPtr->dictSize += (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; - return result; + return LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, withPrefix64k, noDictIssue, acceleration); } /* external dictionary mode */ { int result; - if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); - else - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + if (streamPtr->dictCtx) { + /* We depend here on the fact that dictCtx'es (produced by + * LZ4_loadDict) guarantee that their tables contain no references + * to offsets between dictCtx->currentOffset - 64 KB and + * dictCtx->currentOffset - dictCtx->dictSize. This makes it safe + * to use noDictIssue even when the dict isn't a full 64 KB. + */ + if (inputSize > 4 KB) { + /* For compressing large blobs, it is faster to pay the setup + * cost to copy the dictionary's tables into the active context, + * so that the compression loop is only looking into one table. + */ + LZ4_memcpy(streamPtr, streamPtr->dictCtx, sizeof(*streamPtr)); + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingDictCtx, noDictIssue, acceleration); + } + } else { + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, dictSmall, acceleration); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, usingExtDict, noDictIssue, acceleration); + } + } streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; return result; } } -/* Hidden debug function, to force external dictionary mode */ -int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +/* Hidden debug function, to force-test external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int srcSize) { LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; int result; - const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; - const BYTE* smallest = dictEnd; - if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; - LZ4_renormDictT(streamPtr, smallest); + LZ4_renormDictT(streamPtr, srcSize); - result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, dictSmall, 1); + } else { + result = LZ4_compress_generic(streamPtr, source, dest, srcSize, NULL, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + } streamPtr->dictionary = (const BYTE*)source; - streamPtr->dictSize = (U32)inputSize; - streamPtr->currentOffset += (U32)inputSize; + streamPtr->dictSize = (U32)srcSize; return result; } @@ -1087,10 +1670,12 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; - if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ - if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + if ((U32)dictSize > 64 KB) { dictSize = 64 KB; } /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) { dictSize = (int)dict->dictSize; } - memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + if (safeBuffer == NULL) assert(dictSize == 0); + if (dictSize > 0) + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; @@ -1100,212 +1685,602 @@ int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) -/*-***************************** -* Decompression functions -*******************************/ +/*-******************************* + * Decompression functions + ********************************/ + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; + +#undef MIN +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +/* Read the variable-length literal or match length. + * + * ip - pointer to use as input. + * lencheck - end ip. Return an error if ip advances >= lencheck. + * loop_check - check ip >= lencheck in body of loop. Returns loop_error if so. + * initial_check - check ip >= lencheck before start of loop. Returns initial_error if so. + * error (output) - error code. Should be set to 0 before call. + */ +typedef enum { loop_error = -2, initial_error = -1, ok = 0 } variable_length_error; +LZ4_FORCE_INLINE unsigned +read_variable_length(const BYTE**ip, const BYTE* lencheck, + int loop_check, int initial_check, + variable_length_error* error) +{ + U32 length = 0; + U32 s; + if (initial_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = initial_error; + return length; + } + do { + s = **ip; + (*ip)++; + length += s; + if (loop_check && unlikely((*ip) >= lencheck)) { /* overflow detection */ + *error = loop_error; + return length; + } + } while (s==255); + + return length; +} + /*! LZ4_decompress_generic() : * This generic decompression function covers all use cases. * It shall be instantiated several times, using different sets of directives. * Note that it is important for performance that this function really get inlined, * in order to remove useless branches during compilation optimization. */ -LZ4_FORCE_INLINE int LZ4_decompress_generic( +LZ4_FORCE_INLINE int +LZ4_decompress_generic( const char* const src, char* const dst, int srcSize, int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dst when no prefix */ + endCondition_directive endOnInput, /* endOnOutputSize, endOnInputSize */ + earlyEnd_directive partialDecoding, /* full, partial */ + dict_directive dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ const BYTE* const dictStart, /* only if dict==usingExtDict */ const size_t dictSize /* note : = 0 if noDict */ ) { - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; + if (src == NULL) { return -1; } - BYTE* op = (BYTE*) dst; - BYTE* const oend = op + outputSize; - BYTE* cpy; - BYTE* oexit = op + targetOutputSize; + { const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; - const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; - const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; - const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + outputSize; + BYTE* cpy; - const int safeDecode = (endOnInput==endOnInputSize); - const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + const BYTE* const dictEnd = (dictStart == NULL) ? NULL : dictStart + dictSize; + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); - /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ - if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ - if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); - /* Main Loop : decode sequences */ - while (1) { - size_t length; + /* Set up the "end" pointers for the shortcut. */ + const BYTE* const shortiend = iend - (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; + const BYTE* const shortoend = oend - (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; + const BYTE* match; size_t offset; + unsigned token; + size_t length; - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + /* Fast loop : decode sequences as long as output < iend-FASTLOOP_SAFE_DISTANCE */ + while (1) { + /* Main fastloop assertion: We can always wildcopy FASTLOOP_SAFE_DISTANCE */ + assert(oend - op >= FASTLOOP_SAFE_DISTANCE); + if (endOnInput) { assert(ip < iend); } + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + + /* copy literals */ + cpy = op+length; + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if (endOnInput) { /* LZ4_decompress_safe() */ + if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; } + LZ4_wildCopy32(op, ip, cpy); + } else { /* LZ4_decompress_fast() */ + if (cpy>oend-8) { goto safe_literal_copy; } + LZ4_wildCopy8(op, ip, cpy); /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and only relies on end-of-block properties */ + } + ip += length; op = cpy; } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + cpy = op+length; + if (endOnInput) { /* LZ4_decompress_safe() */ + DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length); + /* We don't need to check oend, since we check it once for each loop below */ + if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; } + /* Literals can only be 14, but hope compilers optimize if we copy by a register size */ + LZ4_memcpy(op, ip, 16); + } else { /* LZ4_decompress_fast() */ + /* LZ4_decompress_fast() cannot copy more than 8 bytes at a time : + * it doesn't know input length, and relies on end-of-block properties */ + LZ4_memcpy(op, ip, 8); + if (length > 8) { LZ4_memcpy(op+8, ip+8, 8); } + } + ip += length; op = cpy; } - memcpy(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; - - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + assert(match <= op); + + /* get matchlength */ + length = token & ML_MASK; + + if (length == ML_MASK) { + variable_length_error error = ok; + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); + if (error != ok) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */ + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + } else { + length += MINMATCH; + if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) { + goto safe_match_copy; + } + + /* Fastpath check: Avoids a branch in LZ4_wildCopy32 if true */ + if ((dict == withPrefix64k) || (match >= lowPrefix)) { + if (offset >= 8) { + assert(match >= lowPrefix); + assert(match <= op); + assert(op + 18 <= oend); + + LZ4_memcpy(op, match, 8); + LZ4_memcpy(op+8, match+8, 8); + LZ4_memcpy(op+16, match+16, 2); + op += length; + continue; + } } } + + if (checkOffset && (unlikely(match + dictSize < lowPrefix))) { goto _output_error; } /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) { + DEBUGLOG(7, "partialDecoding: dictionary match, close to dstEnd"); + length = MIN(length, (size_t)(oend-op)); + } else { + goto _output_error; /* end-of-block condition violated */ + } } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) { *op++ = *copyFrom++; } + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + + /* copy match within block */ + cpy = op + length; + + assert((op <= oend) && (oend-op >= 32)); + if (unlikely(offset<16)) { + LZ4_memcpy_using_offset(op, match, cpy, offset); + } else { + LZ4_wildCopy32(op, match, cpy); + } + + op = cpy; /* wildcopy correction */ } - length += MINMATCH; + safe_decode: +#endif - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + /* Main Loop : decode remaining sequences where output < FASTLOOP_SAFE_DISTANCE */ + while (1) { + token = *ip++; + length = token >> ML_BITS; /* literal length */ + + assert(!endOnInput || ip <= iend); /* ip < iend before the increment */ + + /* A two-stage shortcut for the most common case: + * 1) If the literal length is 0..14, and there is enough space, + * enter the shortcut and copy 16 bytes on behalf of the literals + * (in the fast mode, only 8 bytes can be safely copied this way). + * 2) Further if the match length is 4..18, copy 18 bytes in a similar + * manner; but we ensure that there's enough space in the output for + * those 18 bytes earlier, upon entering the shortcut (in other words, + * there is a combined check for both stages). + */ + if ( (endOnInput ? length != RUN_MASK : length <= 8) + /* strictly "less than" on input, to re-enter the loop with at least one byte */ + && likely((endOnInput ? ip < shortiend : 1) & (op <= shortoend)) ) { + /* Copy the literals */ + LZ4_memcpy(op, ip, endOnInput ? 16 : 8); + op += length; ip += length; + + /* The second stage: prepare for match copying, decode full info. + * If it doesn't work out, the info won't be wasted. */ + length = token & ML_MASK; /* match length */ + offset = LZ4_readLE16(ip); ip += 2; + match = op - offset; + assert(match <= op); /* check overflow */ + + /* Do not deal with overlapping matches. */ + if ( (length != ML_MASK) + && (offset >= 8) + && (dict==withPrefix64k || match >= lowPrefix) ) { + /* Copy the match. */ + LZ4_memcpy(op + 0, match + 0, 8); + LZ4_memcpy(op + 8, match + 8, 8); + LZ4_memcpy(op +16, match +16, 2); + op += length + MINMATCH; + /* Both stages worked, load the next token. */ + continue; + } + + /* The second stage didn't work out, but the info is ready. + * Propel it right to the point of match copying. */ + goto _copy_match; + } - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); + /* decode literal length */ + if (length == RUN_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend-RUN_MASK, (int)endOnInput, (int)endOnInput, &error); + if (error == initial_error) { goto _output_error; } + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)(op))) { goto _output_error; } /* overflow detection */ + if ((safeDecode) && unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */ + } + + /* copy literals */ + cpy = op+length; +#if LZ4_FAST_DEC_LOOP + safe_literal_copy: +#endif + LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); + if ( ((endOnInput) && ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + /* We've either hit the input parsing restriction or the output parsing restriction. + * In the normal scenario, decoding a full block, it must be the last sequence, + * otherwise it's an error (invalid input or dimensions). + * In partialDecoding scenario, it's necessary to ensure there is no buffer overflow. + */ + if (partialDecoding) { + /* Since we are partial decoding we may be in this block because of the output parsing + * restriction, which is not valid since the output buffer is allowed to be undersized. + */ + assert(endOnInput); + DEBUGLOG(7, "partialDecoding: copying literals, close to input or output end") + DEBUGLOG(7, "partialDecoding: literal length = %u", (unsigned)length); + DEBUGLOG(7, "partialDecoding: remaining space in dstBuffer : %i", (int)(oend - op)); + DEBUGLOG(7, "partialDecoding: remaining space in srcBuffer : %i", (int)(iend - ip)); + /* Finishing in the middle of a literals segment, + * due to lack of input. + */ + if (ip+length > iend) { + length = (size_t)(iend-ip); + cpy = op + length; + } + /* Finishing in the middle of a literals segment, + * due to lack of output space. + */ + if (cpy > oend) { + cpy = oend; + assert(op<=oend); + length = (size_t)(oend-op); + } + } else { + /* We must be on the last sequence because of the parsing limitations so check + * that we exactly regenerate the original size (must be exact when !endOnInput). + */ + if ((!endOnInput) && (cpy != oend)) { goto _output_error; } + /* We must be on the last sequence (or invalid) because of the parsing limitations + * so check that we exactly consume the input and don't overrun the output buffer. + */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) { + DEBUGLOG(6, "should have been last run of literals") + DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend); + DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend); + goto _output_error; + } + } + memmove(op, ip, length); /* supports overlapping memory regions; only matters for in-place decompression scenarios */ + ip += length; op += length; + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend-2))) { + break; + } } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - memcpy(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; + LZ4_wildCopy8(op, ip, cpy); /* may overwrite up to WILDCOPYLENGTH beyond cpy */ + ip += length; op = cpy; + } + + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + + /* get matchlength */ + length = token & ML_MASK; + + _copy_match: + if (length == ML_MASK) { + variable_length_error error = ok; + length += read_variable_length(&ip, iend - LASTLITERALS + 1, (int)endOnInput, 0, &error); + if (error != ok) goto _output_error; + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + +#if LZ4_FAST_DEC_LOOP + safe_match_copy: +#endif + if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ + /* match starting within external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) { + if (partialDecoding) length = MIN(length, (size_t)(oend-op)); + else goto _output_error; /* doesn't respect parsing restriction */ + } + + if (length <= (size_t)(lowPrefix-match)) { + /* match fits entirely within external dictionary : just copy */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; } else { - memcpy(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } + /* match stretches into both external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix - match); + size_t const restSize = length - copySize; + LZ4_memcpy(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op - lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + LZ4_memcpy(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } + assert(match >= lowPrefix); + + /* copy match within block */ + cpy = op + length; + + /* partialDecoding : may end anywhere within the block */ + assert(op<=oend); + if (partialDecoding && (cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + size_t const mlen = MIN(length, (size_t)(oend-op)); + const BYTE* const matchEnd = match + mlen; + BYTE* const copyEnd = op + mlen; + if (matchEnd > op) { /* overlap copy */ + while (op < copyEnd) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, mlen); + } + op = copyEnd; + if (op == oend) { break; } + continue; + } - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - memcpy(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; - - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; + if (unlikely(offset<8)) { + LZ4_write32(op, 0); /* silence msan warning when offset==0 */ + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += inc32table[offset]; + LZ4_memcpy(op+4, match, 4); + match -= dec64table[offset]; + } else { + LZ4_memcpy(op, match, 8); + match += 8; } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); + op += 8; + + if (unlikely(cpy > oend-MATCH_SAFEGUARD_DISTANCE)) { + BYTE* const oCopyLimit = oend - (WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) { goto _output_error; } /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy8(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op < cpy) { *op++ = *match++; } + } else { + LZ4_memcpy(op, match, 8); + if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); } + } + op = cpy; /* wildcopy correction */ } - op=cpy; /* correction */ - } - - /* end of decoding */ - if (endOnInput) - return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ - else - return (int) (((const char*)ip)-src); /* Nb of input bytes read */ - /* Overflow error detected */ -_output_error: - return (int) (-(((const char*)ip)-src))-1; + /* end of decoding */ + if (endOnInput) { + DEBUGLOG(5, "decoded %i bytes", (int) (((char*)op)-dst)); + return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ + } else { + return (int) (((const char*)ip)-src); /* Nb of input bytes read */ + } + + /* Overflow error detected */ + _output_error: + return (int) (-(((const char*)ip)-src))-1; + } } +/*===== Instantiate the API decoding functions. =====*/ + +LZ4_FORCE_O2 int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, + endOnInputSize, decode_full_block, noDict, + (BYTE*)dest, NULL, 0); } -int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) +LZ4_FORCE_O2 +int LZ4_decompress_safe_partial(const char* src, char* dst, int compressedSize, int targetOutputSize, int dstCapacity) { - return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); + dstCapacity = MIN(targetOutputSize, dstCapacity); + return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, + endOnInputSize, partial_decode, + noDict, (BYTE*)dst, NULL, 0); } +LZ4_FORCE_O2 int LZ4_decompress_fast(const char* source, char* dest, int originalSize) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); +} + +/*===== Instantiate a few more decoding cases, used more than once. =====*/ + +LZ4_FORCE_O2 /* Exported, an obsolete API function. */ +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, withPrefix64k, + (BYTE*)dest - 64 KB, NULL, 0); } +/* Another obsolete API function, paired with the previous one. */ +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + /* LZ4_decompress_fast doesn't validate match offsets, + * and thus serves well with any prefixed dictionary. */ + return LZ4_decompress_fast(source, dest, originalSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_safe_withSmallPrefix(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, noDict, + (BYTE*)dest-prefixSize, NULL, 0); +} + +LZ4_FORCE_O2 +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, + int compressedSize, int maxOutputSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_O2 +static int LZ4_decompress_fast_extDict(const char* source, char* dest, int originalSize, + const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, usingExtDict, + (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +/* The "double dictionary" mode, for use with e.g. ring buffers: the first part + * of the dictionary is passed as prefix, and the second via dictStart + dictSize. + * These routines are used only once, in LZ4_decompress_*_continue(). + */ +LZ4_FORCE_INLINE +int LZ4_decompress_safe_doubleDict(const char* source, char* dest, int compressedSize, int maxOutputSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} + +LZ4_FORCE_INLINE +int LZ4_decompress_fast_doubleDict(const char* source, char* dest, int originalSize, + size_t prefixSize, const void* dictStart, size_t dictSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, decode_full_block, usingExtDict, + (BYTE*)dest-prefixSize, (const BYTE*)dictStart, dictSize); +} /*===== streaming decompression functions =====*/ LZ4_streamDecode_t* LZ4_createStreamDecode(void) { - LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); + LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOC_AND_ZERO(sizeof(LZ4_streamDecode_t)); + LZ4_STATIC_ASSERT(LZ4_STREAMDECODESIZE >= sizeof(LZ4_streamDecode_t_internal)); /* A compilation error here means LZ4_STREAMDECODESIZE is not large enough */ return lz4s; } int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) { - if (!LZ4_stream) return 0; /* support free on NULL */ + if (LZ4_stream == NULL) { return 0; } /* support free on NULL */ FREEMEM(LZ4_stream); return 0; } -/*! - * LZ4_setStreamDecode() : - * Use this function to instruct where to find the dictionary. - * This function is not necessary if previous data is still available where it was decoded. - * Loading a size of 0 is allowed (same effect as no dictionary). - * Return : 1 if OK, 0 if error +/*! LZ4_setStreamDecode() : + * Use this function to instruct where to find the dictionary. + * This function is not necessary if previous data is still available where it was decoded. + * Loading a size of 0 is allowed (same effect as no dictionary). + * @return : 1 if OK, 0 if error */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) { @@ -1317,6 +2292,25 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti return 1; } +/*! LZ4_decoderRingBufferSize() : + * when setting a ring buffer for streaming decompression (optional scenario), + * provides the minimum size of this ring buffer + * to be compatible with any source respecting maxBlockSize condition. + * Note : in a ring buffer scenario, + * blocks are presumed decompressed next to each other. + * When not enough space remains for next block (remainingSize < maxBlockSize), + * decoding resumes from beginning of ring buffer. + * @return : minimum ring buffer size, + * or 0 if there is an error (invalid maxBlockSize). + */ +int LZ4_decoderRingBufferSize(int maxBlockSize) +{ + if (maxBlockSize < 0) return 0; + if (maxBlockSize > LZ4_MAX_INPUT_SIZE) return 0; + if (maxBlockSize < 16) maxBlockSize = 16; + return LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize); +} + /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. @@ -1324,52 +2318,75 @@ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dicti If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ +LZ4_FORCE_O2 int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (lz4sd->prefixSize == 0) { + /* The first call, no dictionary yet. */ + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (result <= 0) return result; - lz4sd->prefixSize += result; + lz4sd->prefixSize = (size_t)result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + /* They're rolling the current segment. */ + if (lz4sd->prefixSize >= 64 KB - 1) + result = LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + else if (lz4sd->extDictSize == 0) + result = LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize); + else + result = LZ4_decompress_safe_doubleDict(source, dest, compressedSize, maxOutputSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)result; lz4sd->prefixEnd += result; } else { + /* The buffer wraps around, or they're switching to another buffer. */ lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, - endOnInputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = result; + lz4sd->prefixSize = (size_t)result; lz4sd->prefixEnd = (BYTE*)dest + result; } return result; } +LZ4_FORCE_O2 int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; int result; + assert(originalSize >= 0); - if (lz4sd->prefixEnd == (BYTE*)dest) { - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (lz4sd->prefixSize == 0) { + assert(lz4sd->extDictSize == 0); + result = LZ4_decompress_fast(source, dest, originalSize); if (result <= 0) return result; - lz4sd->prefixSize += originalSize; + lz4sd->prefixSize = (size_t)originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } else if (lz4sd->prefixEnd == (BYTE*)dest) { + if (lz4sd->prefixSize >= 64 KB - 1 || lz4sd->extDictSize == 0) + result = LZ4_decompress_fast(source, dest, originalSize); + else + result = LZ4_decompress_fast_doubleDict(source, dest, originalSize, + lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += (size_t)originalSize; lz4sd->prefixEnd += originalSize; } else { lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; - result = LZ4_decompress_generic(source, dest, 0, originalSize, - endOnOutputSize, full, 0, - usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + result = LZ4_decompress_fast_extDict(source, dest, originalSize, + lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; - lz4sd->prefixSize = originalSize; + lz4sd->prefixSize = (size_t)originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } @@ -1384,32 +2401,27 @@ Advanced decoding functions : the dictionary must be explicitly provided within parameters */ -LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { if (dictSize==0) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + return LZ4_decompress_safe(source, dest, compressedSize, maxOutputSize); if (dictStart+dictSize == dest) { - if (dictSize >= (int)(64 KB - 1)) - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + if (dictSize >= 64 KB - 1) { + return LZ4_decompress_safe_withPrefix64k(source, dest, compressedSize, maxOutputSize); + } + assert(dictSize >= 0); + return LZ4_decompress_safe_withSmallPrefix(source, dest, compressedSize, maxOutputSize, (size_t)dictSize); } - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); -} - -int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); + assert(dictSize >= 0); + return LZ4_decompress_safe_forceExtDict(source, dest, compressedSize, maxOutputSize, dictStart, (size_t)dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { - return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); -} - -/* debug function */ -int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); + if (dictSize==0 || dictStart+dictSize == dest) + return LZ4_decompress_fast(source, dest, originalSize); + assert(dictSize >= 0); + return LZ4_decompress_fast_extDict(source, dest, originalSize, dictStart, (size_t)dictSize); } @@ -1417,64 +2429,67 @@ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compres * Obsolete Functions ***************************************************/ /* obsolete compression functions */ -int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } -int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } -int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } -int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } -int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } -int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_default(source, dest, inputSize, maxOutputSize); +} +int LZ4_compress(const char* src, char* dest, int srcSize) +{ + return LZ4_compress_default(src, dest, srcSize, LZ4_compressBound(srcSize)); +} +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); +} +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) +{ + return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); +} +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int dstCapacity) +{ + return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, dstCapacity, 1); +} +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) +{ + return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); +} /* -These function names are deprecated and should no longer be used. +These decompression functions are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. - LZ4_uncompress is totally equivalent to LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe */ -int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } -int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } - +int LZ4_uncompress (const char* source, char* dest, int outputSize) +{ + return LZ4_decompress_fast(source, dest, outputSize); +} +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) +{ + return LZ4_decompress_safe(source, dest, isize, maxOutputSize); +} /* Obsolete Streaming functions */ -int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } - -static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) -{ - MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); - lz4ds->internal_donotuse.bufferStart = base; -} +int LZ4_sizeofStreamState(void) { return LZ4_STREAMSIZE; } int LZ4_resetStreamState(void* state, char* inputBuffer) { - if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ - LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); + (void)inputBuffer; + LZ4_resetStream((LZ4_stream_t*)state); return 0; } void* LZ4_create (char* inputBuffer) { - LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); - LZ4_init (lz4ds, (BYTE*)inputBuffer); - return lz4ds; + (void)inputBuffer; + return LZ4_createStream(); } -char* LZ4_slideInputBuffer (void* LZ4_Data) -{ - LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; - int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); - return (char*)(ctx->bufferStart + dictSize); -} - -/* Obsolete streaming decompression functions */ - -int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) -{ - return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); -} - -int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +char* LZ4_slideInputBuffer (void* state) { - return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); + /* avoid const char * -> char * conversion warning */ + return (char *)(uptrval)((LZ4_stream_t*)state)->internal_donotuse.dictionary; } #endif /* LZ4_COMMONDEFS_ONLY */ diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md index 24654ed407ea121c627e0488888a455f9a858646..a16154443c96cfd31cbc7c5d4b49caf3ccbeab9e 100644 --- a/documentation20/cn/00.index/docs.md +++ b/documentation20/cn/00.index/docs.md @@ -131,7 +131,7 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 * [TDengine写入性能测试工具](https://www.taosdata.com/blog/2020/01/18/1166.html) * [IDEA数据库管理工具可视化使用TDengine](https://www.taosdata.com/blog/2020/08/27/1767.html) * [基于Electron开发的跨平台TDengine图形化管理工具](https://github.com/skye0207/TDengineGUI) -* [DataX，支持TDengine的离线数据采集/同步工具](https://github.com/wgzhao/DataX)（文档：[读取插件](https://github.com/wgzhao/DataX/blob/master/docs/src/main/sphinx/reader/tdenginereader.md)、[写入插件](https://github.com/wgzhao/DataX/blob/master/docs/src/main/sphinx/writer/tdenginewriter.md)） +* [基于DataX的TDeninge数据迁移工具](https://www.taosdata.com/blog/2021/10/26/3156.html) ## TDengine与其他数据库的对比测试 diff --git a/documentation20/cn/03.architecture/docs.md b/documentation20/cn/03.architecture/docs.md index 6afdfb7a7d50533e094e13195c21b223daf888d1..5eafea00c8ca84dff466d835e3016d5818e2a1d5 100644 --- a/documentation20/cn/03.architecture/docs.md +++ b/documentation20/cn/03.architecture/docs.md @@ -382,17 +382,17 @@ dataDir [path] 各级存储之间的数据流向：0 级存储 -> 1 级存储 -> 2 级存储。 同一存储等级可挂载多个硬盘，同一存储等级上的数据文件分布在该存储等级的所有硬盘上。 需要说明的是，数据在不同级别的存储介质上的移动，是由系统自动完成的，用户无需干预。 -- primary: 是否为主挂载点，0（是）或 1（否），省略默认为 1。 +- primary: 是否为主挂载点，0（否）或 1（是），省略默认为 1。 -在配置中，只允许一个主挂载点的存在（level=0, primary=0），例如采用如下的配置方式： +在配置中，只允许一个主挂载点的存在（level=0, primary=1），例如采用如下的配置方式： ``` -dataDir /mnt/data1 0 0 -dataDir /mnt/data2 0 1 -dataDir /mnt/data3 1 1 -dataDir /mnt/data4 1 1 -dataDir /mnt/data5 2 1 -dataDir /mnt/data6 2 1 +dataDir /mnt/data1 0 1 +dataDir /mnt/data2 0 0 +dataDir /mnt/data3 1 0 +dataDir /mnt/data4 1 0 +dataDir /mnt/data5 2 0 +dataDir /mnt/data6 2 0 ``` 注意： diff --git a/documentation20/cn/05.insert/docs.md b/documentation20/cn/05.insert/docs.md index 57a7521738a857ee681bfaf38858db14f34abe91..9a0e9b388e639d5e6c6e5094682f07a223c01ada 100644 --- a/documentation20/cn/05.insert/docs.md +++ b/documentation20/cn/05.insert/docs.md @@ -243,7 +243,7 @@ repeater 部分添加 { host:'', port: ### Schemaless 方式写入接口 -除了使用 SQL 方式或者使用参数绑定 API 写入数据外，还可以使用 Schemaless 的方式完成写入。Schemaless 可以免于预先创建超级表/数据子表的数据结构，而是可以直接写入数据，TDengine 系统会根据写入的数据内容自动创建和维护所需要的表结构。Schemaless 的使用方式详见 [Schemaless 写入](https://www.taosdata.com/cn/documentation/insert#schemaless) 章节，这里介绍与之配套使用的 C/C++ API。 - -2.2.0.0版本接口： -- `int taos_insert_lines(TAOS* taos, char* lines[], int numLines)` - - 以 Schemaless 格式写入多行数据。其中： - * taos：调用 taos_connect 返回的数据库连接。 - * lines：由 char 字符串指针组成的数组，指向本次想要写入数据库的多行数据。 - * numLines：lines 数据的总行数。 - - 返回值为 0 表示写入成功，非零值表示出错。具体错误代码请参见 [taoserror.h](https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h) 文件。 - - 说明： - 1. 此接口是一个同步阻塞式接口，使用时机与 `taos_query()` 一致。 - 2. 在调用此接口之前，必须先调用 `taos_select_db()` 来确定目前是在向哪个 DB 来写入。 - -2.3.0.0版本接口： -- `int taos_schemaless_insert(TAOS* taos, const char* lines[], int numLines, int protocol, const char* precision, int* affectedRows, char* msg, int msgBufLen)` - **参数说明** - taos: 数据库连接，通过taos_connect 函数建立的数据库连接。 - lines：文本数据。满足解析格式要求的无模式文本字符串。 - numLines:文本数据的行数，不能为 0 。 - protocol: 行协议类型，用于标识文本数据格式。 - precision：文本数据中的时间戳精度字符串。 - affectedRows：插入操作完成以后，正确写入到数据库中的记录行数。 - msg: 如果出现错误（函数返回值不为 0）情况下，错误提示信息。该参数是输入参数，需要用户指定消息输出缓冲区，如果不指定该缓冲区（输入为NULL），即使出现错误也不会得到错误提示信息。 - msgBufLen: 缓冲区的长度，避免错误提示消息越界。 - - **返回值** - 0：无错误发生。 - 非 0 值：发生了错误。此时可以通过msg获取错误信息的提示。该返回值含义可以参考taoserror.h文件中的错误码定义。 +除了使用 SQL 方式或者使用参数绑定 API 写入数据外，还可以使用 Schemaless 的方式完成写入。Schemaless 可以免于预先创建超级表/数据子表的数据结构，而是可以直接写入数据，TDengine 系统会根据写入的数据内容自动创建和维护所需要的表结构。Schemaless 的使用方式详见 [Schemaless 写入](https://www.taosdata.com/cn/documentation/insert#schemaless) 章节，这里介绍与之配套使用的 C/C++ API。 + +- `TAOS_RES* taos_schemaless_insert(TAOS* taos, const char* lines[], int numLines, int protocol, int precision)` - **说明** - 协议类型是枚举类型，包含以下三种格式： - SML_LINE_PROTOCOL：InfluxDB行协议（Line Protocol) - SML_TELNET_PROTOCOL: OpenTSDB文本行协议 - SML_JSON_PROTOCOL: OpenTSDB Json协议格式 + **功能说明** + 该接口将行协议的文本数据写入到TDengine中。 + + **参数说明** + taos: 数据库连接，通过taos_connect 函数建立的数据库连接。 + lines：文本数据。满足解析格式要求的无模式文本字符串。 + numLines:文本数据的行数，不能为 0 。 + protocol: 行协议类型，用于标识文本数据格式。 + precision：文本数据中的时间戳精度字符串。 + + **返回值** + TAOS_RES 结构体，应用可以通过使用 taos_errstr 获得错误信息，也可以使用 taos_errno 获得错误码。 + 在某些情况下，返回的 TAOS_RES 为 NULL，此时仍然可以调用 taos_errno 来安全地获得错误码信息。 + 返回的 TAOS_RES 需要调用方来负责释放，否则会出现内存泄漏。 + + **说明** + 协议类型是枚举类型，包含以下三种格式： + TSDB_SML_LINE_PROTOCOL：InfluxDB行协议（Line Protocol) + TSDB_SML_TELNET_PROTOCOL: OpenTSDB文本行协议 + TSDB_SML_JSON_PROTOCOL: OpenTSDB Json协议格式 + + 时间戳分辨率的定义，定义在 taos.h 文件中，具体内容如下： + TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0, + TSDB_SML_TIMESTAMP_HOURS, + TSDB_SML_TIMESTAMP_MINUTES, + TSDB_SML_TIMESTAMP_SECONDS, + TSDB_SML_TIMESTAMP_MILLI_SECONDS, + TSDB_SML_TIMESTAMP_MICRO_SECONDS, + TSDB_SML_TIMESTAMP_NANO_SECONDS - 时间戳分辨率的说明使用如下字符串：“h“ （小时）、”m“（分钟）、”s“ （秒） ”ms“（毫秒）、”u“ (微秒）、”ns”（纳秒），不区分大小写。需要注意的是，时间戳分辨率参数只在协议类型为 SML_LINE_PROTOCOL 的时候生效。对于 OpenTSDB的文本协议，时间戳的解析遵循其官方解析规则 — 按照时间戳包含的字符的数量来确认时间精度。 + 需要注意的是，时间戳分辨率参数只在协议类型为 SML_LINE_PROTOCOL 的时候生效。 + 对于 OpenTSDB 的文本协议，时间戳的解析遵循其官方解析规则 — 按照时间戳包含的字符的数量来确认时间精度。 + + **支持版本** + 该功能接口从2.3.0.0版本开始支持。 ```c #include @@ -454,10 +454,7 @@ int main() { const char* host = "127.0.0.1"; const char* user = "root"; const char* passwd = "taosdata"; - - // error message buffer - char msg[512] = {0}; - + // connect to server TAOS* taos = taos_connect(host, user, passwd, "test", 0); @@ -468,17 +465,18 @@ int main() { }; // schema-less insert - int code = taos_schemaless_insert(taos, lines1, 2, SML_LINE_PROTOCOL, "ns", msg, sizeof(msg)/sizeof(msg[0])); - if (code != 0) { - printf("failed to insert schema-less data, reason: %s\n", msg); + TAOS_RES* res = taos_schemaless_insert(taos, lines1, 2, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + if (taos_errno(res) != 0) { + printf("failed to insert schema-less data, reason: %s\n", taos_errstr(res)); } + taos_free_result(res); + // close the connection taos_close(taos); return (code); } ``` -**注**：后续2.2.0.0版本也更新成2.3.0.0版本的接口。 ### 连续查询接口 @@ -575,6 +573,14 @@ cd C:\TDengine\connector\python python -m pip install . ``` +**PyPI** + +从2.1.1版本开始，用户可以从[PyPI](https://pypi.org/project/taospy/)安装： + +```sh +pip install taospy +``` + * 如果机器上没有pip命令，用户可将src/connector/python下的taos文件夹拷贝到应用程序的目录使用。 对于windows 客户端，安装TDengine windows 客户端后，将C:\TDengine\driver\taos.dll拷贝到C:\windows\system32目录下即可。 @@ -610,6 +616,22 @@ python3 PythonChecker.py -host ### Python连接器的使用 +#### PEP-249 兼容API + +您可以像其他数据库一样，使用类似 [PEP-249](https://www.python.org/dev/peps/pep-0249/) 数据库API规范风格的API： + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +``` + #### 代码示例 * 导入TDengine客户端模块 @@ -665,6 +687,44 @@ for data in c1: print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])) ``` +* 从v2.1.0版本开始, 我们提供另外一种API：`connection.query` + + ```python + import taos + + conn = taos.connect() + conn.execute("create database if not exists pytest") + + result = conn.query("show databases") + num_of_fields = result.field_count + for field in result.fields: + print(field) + for row in result: + print(row) + conn.execute("drop database pytest") + ``` + + `query` 方法会返回一个 `TaosResult` 类对象，并提供了以下有用的属性或方法: + + 属性: + + - `fields`: `TaosFields` 集合类，提供返回数据的列信息。 + - `field_count`: 返回数据的列数. + - `affected_rows`: 插入数据的行数. + - `row_count`: 查询数据结果数. + - `precision`: 当前数据库的时间精度. + + 方法: + + - `fetch_all()`: 类似于 `cursor.fetchall()` 返回同样的集合数据 + - `fetch_all_into_dict()`: v2.1.1 新添加的API，将上面的数据转换成字典类型返回 + - `blocks_iter()` `rows_iter()`: 根据底层API提供的两种不同迭代器。 + - `fetch_rows_a`: 异步API + - `errno`: 错误码 + - `errstr`: 错误信息 + - `close`: 关闭结果对象，一般不需要直接调用 + + * 创建订阅 ```python diff --git a/documentation20/cn/12.taos-sql/02.udf/docs.md b/documentation20/cn/12.taos-sql/02.udf/docs.md index 454f650b111ac02f318c6f2bdd9bf8eb9b3f3e5d..5b068d43fda8d765c052582dc1bdda163d9d72e3 100644 --- a/documentation20/cn/12.taos-sql/02.udf/docs.md +++ b/documentation20/cn/12.taos-sql/02.udf/docs.md @@ -1,6 +1,6 @@ # UDF（用户定义函数） -在有些应用场景中，应用逻辑需要的查询无法直接使用系统内置的函数来表示。利用 UDF 功能，TDengine 可以插入用户编写的处理代码并在查询中使用它们，就能够很方便地解决特殊应用场景中的使用需求。 +在有些应用场景中，应用逻辑需要的查询无法直接使用系统内置的函数来表示。利用 UDF 功能，TDengine 可以插入用户编写的处理代码并在查询中使用它们，就能够很方便地解决特殊应用场景中的使用需求。 UDF 通常以数据表中的一列数据做为输入，同时支持以嵌套子查询的结果作为输入。 从 2.2.0.0 版本开始，TDengine 支持通过 C/C++ 语言进行 UDF 定义。接下来结合示例讲解 UDF 的使用方法。 @@ -9,76 +9,70 @@ TDengine 提供 3 个 UDF 的源代码示例，分别为： * [add_one.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/add_one.c) * [abs_max.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/abs_max.c) -* [sum_double.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/sum_double.c) +* [demo.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/demo.c) -### 无需中间变量的标量函数 +### 标量函数 -[add_one.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/add_one.c) 是结构最简单的 UDF 实现。其功能为：对传入的一个数据列（可能因 WHERE 子句进行了筛选）中的每一项，都输出 +1 之后的值，并且要求输入的列数据类型为 INT。 +[add_one.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/add_one.c) 是结构最简单的 UDF 实现。其功能为：对传入的一个数据列（可能因 WHERE 子句进行了筛选）中的每一项，都输出 +1 之后的值，并且要求输入的列数据类型为 INT。 -这一具体的处理逻辑在函数 `void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput, int* numOfOutput, short otype, short obytes, SUdfInit* buf)` 中定义。这类用于实现 UDF 的基础计算逻辑的函数，我们称为 udfNormalFunc，也就是对行数据块的标量计算函数。需要注意的是，udfNormalFunc 的参数项是固定的，用于按照约束完成与引擎之间的数据交换。 +这一具体的处理逻辑在函数 `void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, int* numOfOutput, short otype, short obytes, SUdfInit* buf)` 中定义。这类用于实现 UDF 的基础计算逻辑的函数，我们称为 udfNormalFunc，也就是对行数据块的标量计算函数。需要注意的是，udfNormalFunc 的参数项是固定的，用于按照约束完成与引擎之间的数据交换。 - udfNormalFunc 中各参数的具体含义是： - * data：存有输入的数据。 + * data：输入数据。 * itype：输入数据的类型。这里采用的是短整型表示法，与各种数据类型对应的值可以参见 [column_meta 中的列类型说明](https://www.taosdata.com/cn/documentation/connector#column_meta)。例如 4 用于表示 INT 型。 * iBytes：输入数据中每个值会占用的字节数。 * numOfRows：输入数据的总行数。 - * ts：主键时间戳在输入中的列数据。 - * dataOutput：输出数据的缓冲区。 - * interBuf：系统使用的中间临时缓冲区，通常用户逻辑无需对 interBuf 进行处理。 - * tsOutput：主键时间戳在输出时的列数据。 - * numOfOutput：输出数据的个数。 + * ts：主键时间戳在输入中的列数据(只读)。 + * dataOutput：输出数据的缓冲区，缓冲区大小为用户指定的输出类型大小 * numOfRows。 + * interBuf：中间计算结果的缓冲区，大小为用户在创建 UDF 时指定的BUFSIZE大小。通常用于计算中间结果与最终结果不一致时使用，由引擎负责分配与释放。 + * tsOutput：主键时间戳在输出时的列数据，如果非空可用于输出结果对应的时间戳。 + * numOfOutput：输出结果的个数（行数）。 * oType：输出数据的类型。取值含义与 itype 参数一致。 - * oBytes：输出数据中每个值会占用的字节数。 - * buf：计算过程的中间变量缓冲区。 + * oBytes：输出数据中每个值占用的字节数。 + * buf：用于在 UDF 与引擎间的状态控制信息传递块。 -其中 buf 参数需要用到一个自定义结构体 SUdfInit。在这个例子中，因为 add_one 的计算过程无需用到中间变量缓存，所以可以把 SUdfInit 定义成一个空结构体。 -### 无需中间变量的聚合函数 +### 聚合函数 [abs_max.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/abs_max.c) 实现的是一个聚合函数，功能是对一组数据按绝对值取最大值。 -其计算过程为：与所在查询语句相关的数据会被分为多个行数据块，对每个行数据块调用 udfNormalFunc（在本例的实现代码中，实际函数名是 `abs_max`)，再将每个数据块的计算结果调用 udfMergeFunc（本例中，其实际的函数名是 `abs_max_merge`）进行聚合，生成每个子表的聚合结果。如果查询指令涉及超级表，那么最后还会通过 udfFinalizeFunc（本例中，其实际的函数名是 `abs_max_finalize`）再把子表的计算结果聚合为超级表的计算结果。 +其计算过程为：与所在查询语句相关的数据会被分为多个行数据块，对每个行数据块调用 udfNormalFunc（在本例的实现代码中，实际函数名是 `abs_max`)来生成每个子表的中间结果，再将子表的中间结果调用 udfMergeFunc（本例中，其实际的函数名是 `abs_max_merge`）进行聚合，生成超级表的最终聚合结果或中间结果。聚合查询最后还会通过 udfFinalizeFunc（本例中，其实际的函数名是 `abs_max_finalize`）再把超级表的中间结果处理为最终结果，最终结果只能含0或1条结果数据。 值得注意的是，udfNormalFunc、udfMergeFunc、udfFinalizeFunc 之间，函数名约定使用相同的前缀，此前缀即 udfNormalFunc 的实际函数名。udfMergeFunc 的函数名后缀 `_merge`、udfFinalizeFunc 的函数名后缀 `_finalize`，是 UDF 实现规则的一部分，系统会按照这些函数名后缀来调用相应功能。 -- udfMergeFunc 用于对计算中间结果进行聚合。本例中 udfMergeFunc 对应的实现函数为 `void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf)`，其中各参数的具体含义是： - * data：udfNormalFunc 的输出组合在一起的数据，也就成为了 udfMergeFunc 的输入。 +- udfMergeFunc 用于对计算中间结果进行聚合，只有针对超级表的聚合查询才需要调用该函数。本例中 udfMergeFunc 对应的实现函数为 `void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf)`，其中各参数的具体含义是： + * data：udfNormalFunc 的输出数据数组，如果使用了 interBuf 那么 data 就是 interBuf 的数组。 * numOfRows：data 中数据的行数。 - * dataOutput：输出数据的缓冲区。 - * numOfOutput：输出数据的个数。 - * buf：计算过程的中间变量缓冲区。 - -- udfFinalizeFunc 用于对计算结果进行最终聚合。本例中 udfFinalizeFunc 对应的实现函数为 `void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf)`，其中各参数的具体含义是： - * dataOutput：输出数据的缓冲区。对 udfFinalizeFunc 来说，其输入数据也来自于这里。 - * interBuf：系统使用的中间临时缓冲区，与 udfNormalFunc 中的同名参数含义一致。 - * numOfOutput：输出数据的个数。 - * buf：计算过程的中间变量缓冲区。 + * dataOutput：输出数据的缓冲区，大小等于一条最终结果的大小。如果此时输出还不是最终结果，可以选择输出到 interBuf 中即data中。 + * numOfOutput：输出结果的个数（行数）。 + * buf：用于在 UDF 与引擎间的状态控制信息传递块。 -同样因为 abs_max 的计算过程无需用到中间变量缓存，所以同样是可以把 SUdfInit 定义成一个空结构体。 - -### 使用中间变量的聚合函数 +- udfFinalizeFunc 用于对计算结果进行最终计算，通常用于有 interBuf 使用的场景。本例中 udfFinalizeFunc 对应的实现函数为 `void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf)`，其中各参数的具体含义是： + * dataOutput：输出数据的缓冲区。 + * interBuf：中间结算结果缓冲区，可作为输入。 + * numOfOutput：输出数据的个数，对聚合函数来说只能是0或者1。 + * buf：用于在 UDF 与引擎间的状态控制信息传递块。 -[sum_double.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/sum_double.c) 也是一个聚合函数，功能是对一组数据输出求和结果的倍数。 -出于功能演示的目的，在这个用户定义函数的实现方法中，用到了中间变量缓冲区 buf。因此，在这个源代码文件中，SUdfInit 就不再是一个空的结构体，而是定义了缓冲区的具体存储内容。 +### 其他 UDF 函数 -也正是因为用到了中间变量缓冲区，因此就需要对这一缓冲区进行初始化和资源释放。具体来说，也即对应 udfInitFunc（本例中，其实际的函数名是 `sum_double_init`）和 udfDestroyFunc（本例中，其实际的函数名是 `sum_double_destroy`）。其函数名命名规则同样是采取以 udfNormalFunc 的实际函数名为前缀，以 `_init` 和 `_destroy` 为后缀。系统会在初始化和资源释放时调用对应名称的函数。 +用户 UDF 程序除了需要实现上面几个函数外，还有两个用于初始化和释放 UDF 与引擎间的状态控制信息传递块的函数。具体来说，也即对应 udfInitFunc 和 udfDestroyFunc。其函数名命名规则同样是采取以 udfNormalFunc 的实际函数名为前缀，以 `_init` 和 `_destroy` 为后缀。系统会在初始化和资源释放时调用对应名称的函数。 -- udfInitFunc 用于初始化中间变量缓冲区中的变量和内容。本例中 udfInitFunc 对应的实现函数为 `int sum_double_init(SUdfInit* buf)`，其中各参数的具体含义是： - * buf：计算过程的中间变量缓冲区。 +- udfInitFunc 用于初始化状态控制信息传递块。上例中 udfInitFunc 对应的实现函数为 `int abs_max_init(SUdfInit* buf)`，其中各参数的具体含义是： + * buf：用于在 UDF 与引擎间的状态控制信息传递块。 -- udfDestroyFunc 用于释放中间变量缓冲区中的变量和内容。本例中 udfDestroyFunc 对应的实现函数为 `void sum_double_destroy(SUdfInit* buf)`，其中各参数的具体含义是： - * buf：计算过程的中间变量缓冲区。 +- udfDestroyFunc 用于释放状态控制信息传递块。上例中 udfDestroyFunc 对应的实现函数为 `void abs_max_destroy(SUdfInit* buf)`，其中各参数的具体含义是： + * buf：用于在 UDF 与引擎间的状态控制信息传递块。 -注意，UDF 的实现过程中需要小心处理对中间变量缓冲区的使用，如果使用不当则有可能导致内存泄露或对资源的过度占用，甚至导致系统服务进程崩溃等。 +目前该功能暂时没有实际意义，待后续扩展使用。 ### UDF 实现方式的规则总结 -根据所要实现的 UDF 类型不同，用户所要实现的功能函数内容也会有所区别： -* 无需中间变量的标量函数：结构体 SUdfInit 可以为空，需实现 udfNormalFunc。 -* 无需中间变量的聚合函数：结构体 SUdfInit 可以为空，需实现 udfNormalFunc、udfMergeFunc、udfFinalizeFunc。 -* 使用中间变量的标量函数：结构体 SUdfInit 需要具体定义，并需实现 udfNormalFunc、udfInitFunc、udfDestroyFunc。 -* 使用中间变量的聚合函数：结构体 SUdfInit 需要具体定义，并需实现 udfNormalFunc、udfInitFunc、udfDestroyFunc、udfMergeFunc、udfFinalizeFunc。 +根据 UDF 函数类型的不同，用户所要实现的功能函数也不同： +* 标量函数：UDF 中需实现 udfNormalFunc。 +* 聚合函数：UDF 中需实现 udfNormalFunc、udfMergeFunc（对超级表查询）、udfFinalizeFunc。 + +需要注意的是，如果对应的函数不需要具体的功能，也需要实现一个空函数。 ## 编译 UDF @@ -97,28 +91,30 @@ gcc -g -O0 -fPIC -shared add_one.c -o add_one.so 用户可以通过 SQL 指令在系统中加载客户端所在主机上的 UDF 函数库（不能通过 RESTful 接口或 HTTP 管理界面来进行这一过程）。一旦创建成功，则当前 TDengine 集群的所有用户都可以在 SQL 指令中使用这些函数。UDF 存储在系统的 MNode 节点上，因此即使重启 TDengine 系统，已经创建的 UDF 也仍然可用。 -在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。 +在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。此外， UDF 支持输入与输出类型不一致，用户需要保证输入数据类型与 UDF 程序匹配，UDF 输出数据类型与 OUTPUTTYPE 匹配。 -- 创建标量函数：`CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize B;` +- 创建标量函数：`CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) [ BUFSIZE B ];` * ids(X)：标量函数未来在 SQL 指令中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - * ids(Y)：包含 UDF 函数实现的动态链接库的库文件路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + * ids(Y)：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； * typename(Z)：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - * B：系统使用的中间临时缓冲区大小，单位是字节，最小 0，最大 512，通常可以设置为 128。 + * B：中间计算结果的缓冲区大小，单位是字节，最小 0，最大 512，如果不使用可以不设置。 例如，如下语句可以把 add_one.so 创建为系统中可用的 UDF： ```sql CREATE FUNCTION add_one AS "/home/taos/udf_example/add_one.so" OUTPUTTYPE INT; ``` -- 创建聚合函数：`CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize B;` +- 创建聚合函数：`CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) [ BUFSIZE B ];` * ids(X)：聚合函数未来在 SQL 指令中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - * ids(Y)：包含 UDF 函数实现的动态链接库的库文件路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + * ids(Y)：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； * typename(Z)：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - * B：系统使用的中间临时缓冲区大小，单位是字节，最小 0，最大 512，通常可以设置为 128。 + * B：中间计算结果的缓冲区大小，单位是字节，最小 0，最大 512，如果不使用可以不设置。 - 例如，如下语句可以把 abs_max.so 创建为系统中可用的 UDF： + 关于中间计算结果的使用，可以参考示例程序[demo.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/demo.c) + + 例如，如下语句可以把 demo.so 创建为系统中可用的 UDF： ```sql - CREATE AGGREGATE FUNCTION abs_max AS "/home/taos/udf_example/abs_max.so" OUTPUTTYPE BIGINT bufsize 128; + CREATE AGGREGATE FUNCTION demo AS "/home/taos/udf_example/demo.so" OUTPUTTYPE DOUBLE bufsize 14; ``` ### 管理 UDF @@ -140,7 +136,7 @@ SELECT X(c) FROM table/stable; 在当前版本下，使用 UDF 存在如下这些限制： 1. 在创建和调用 UDF 时，服务端和客户端都只支持 Linux 操作系统； -2. UDF 不能与系统内建的 SQL 函数混合使用； +2. UDF 不能与系统内建的 SQL 函数混合使用，暂不支持在一条 SQL 语句中使用多个不同名的 UDF ； 3. UDF 只支持以单个数据列作为输入； 4. UDF 只要创建成功，就会被持久化存储到 MNode 节点中； 5. 无法通过 RESTful 接口来创建 UDF； diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 581935a6a3236f6be32996b0f4d47f5186ad7cc3..64020208abe45d589058414fb123d1616c67f2c7 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -171,6 +171,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传 4) 子表名只能由字母、数字和下划线组成，且不能以数字开头，不区分大小写 5) 使用数据类型 binary 或 nchar，需指定其最长的字节数，如 binary(20)，表示 20 字节； + 6) 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 需要注意的是转义字符中的内容必须是可打印字符。 @@ -1578,11 +1579,11 @@ SELECT function_list FROM stb_name CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT); ``` -针对智能电表采集的数据，以 10 分钟为一个阶段，计算过去 24 小时的电流数据的平均值、最大值、电流的中位数、以及随着时间变化的电流走势拟合直线。如果没有计算值，用前一个非 NULL 值填充。使用的查询语句如下： +针对智能电表采集的数据，以 10 分钟为一个阶段，计算过去 24 小时的电流数据的平均值、最大值、电流的中位数。如果没有计算值，用前一个非 NULL 值填充。使用的查询语句如下： ```mysql -SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), PERCENTILE(current, 50) FROM meters - WHERE ts>=NOW-1d +SELECT AVG(current), MAX(current), APERCENTILE(current, 50) FROM meters + WHERE ts>=NOW-1d and ts<=now INTERVAL(10m) FILL(PREV); ``` diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index a3b60baca6d2927ba2015e5f734b2b4d569ac318..7483c972eebe26d0b010724ea699cd94906f382c 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -183,9 +183,10 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端 | TCP | 6035 | 多节点集群的节点间通讯。 | 随 serverPort 端口变化。 | | TCP | 6040 | 多节点集群的节点间数据同步。 | 随 serverPort 端口变化。 | | TCP | 6041 | 客户端与服务端之间的 RESTful 通讯。 | 随 serverPort 端口变化。 | -| TCP | 6042 | Arbitrator 的服务端口。 | 随 Arbitrator 启动参数设置变化。 | -| TCP | 6043 | 支持 collectd 数据接入端口。 | 随 BLM3 启动参数设置变化（2.3.0.1+以上版本）。 | -| TCP | 6044 | 支持 StatsD 的数据接入端口。 | 随 BLM3 启动参数设置变化（2.3.0.1+以上版本）。 | +| TCP | 6042 | Arbitrator 的服务端口。 | 随 Arbitrator 启动参数设置变化。 | +| TCP | 6043 | TaosKeeper 监控服务端口。 | 随 TaosKeeper 启动参数设置变化。 | +| TCP | 6044 | 支持 StatsD 的数据接入端口。 | 随 BLM3 启动参数设置变化（2.3.0.1+以上版本）。 | +| TCP | 6045 | 支持 collectd 数据接入端口。 | 随 BLM3 启动参数设置变化（2.3.0.1+以上版本）。 | | TCP | 6060 | 企业版内 Monitor 服务的网络端口。 | | | UDP | 6030-6034 | 客户端与服务端之间通讯。 | 随 serverPort 端口变化。 | | UDP | 6035-6039 | 多节点集群的节点间通讯。 | 随 serverPort 端口变化。 | @@ -193,12 +194,14 @@ TDengine 中时间戳的时区总是由客户端进行处理，而与服务端 ## 20. go 语言编写组件编译失败怎样解决？ 新版本 TDengine 2.3.0.0 包含一个使用 go 语言开发的 BLM3 组件，取代之前内置的 httpd ，提供包含原 httpd 功能以及支持多种其他软件（Prometheus、Telegraf、collectd、StatsD等）的数据接入功能。 -使用最新 develop 分支代码编译需要先 git submodule update --init --recursive 下载 blm3 仓库代码后再编译。 +使用最新 develop 分支代码编译需要先 `git submodule update --init --recursive` 下载 blm3 仓库代码后再编译。 目前编译方式默认自动编译 blm3。go 语言版本要求 1.14 以上，如果发生 go 编译错误，往往是国内访问 go mod 问题，可以通过设置 go 环境变量来解决： + +```sh go env -w GO111MODULE=on go env -w GOPROXY=https://goproxy.cn,direct +``` 如果希望继续使用之前的内置 httpd，可以关闭 blm3 编译，使用 -cmake .. -DBUILD_HTTP=true 使用原来内置的 httpd。 - +`cmake .. -DBUILD_HTTP=true` 使用原来内置的 httpd。 diff --git a/documentation20/cn/14.devops/02.collectd/docs.md b/documentation20/cn/14.devops/02.collectd/docs.md index 4cd2dcf21442044403a34366e99f700ab583a9bb..2a031d63e55ed7888332757170b781beae787ff7 100644 --- a/documentation20/cn/14.devops/02.collectd/docs.md +++ b/documentation20/cn/14.devops/02.collectd/docs.md @@ -57,7 +57,7 @@ repeater 部分添加 { host:'', port: cmd command line interface: +With Windows TDengine client installed, copy the file "C:\TDengine\driver\taos.dll" to the "C:\Windows\system32" directory and enter the Windows *cmd* command line interface: ```cmd cd C:\TDengine\connector\python python -m pip install . ``` +Or install from PyPI: + +```cmd +pip install taospy +``` + - If there is no `pip` command on the machine, the user can copy the taos folder under src/connector/python to the application directory for use. For Windows client, after installing the TDengine Windows client, copy C:\ TDengine\driver\taos.dll to the C:\ windows\ system32 directory. ### How to use +#### PEP-249 Python Database API + +Definitely you can use the [PEP-249](https://www.python.org/dev/peps/pep-0249/) database API like other type of databases: + +```python +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("show databases") +results = cursor.fetchall() +for row in results: + print(row) +``` + #### Code sample - Import the TDengine client module @@ -488,6 +516,44 @@ for data in c1: print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])) ``` +- Since v2.1.0, python connector provides a new API for query: + +```python +import taos + +conn = taos.connect() +conn.execute("create database if not exists pytest") + +result = conn.query("show databases") +num_of_fields = result.field_count +for field in result.fields: + print(field) +for row in result: + print(row) +conn.execute("drop database pytest") +``` + +The `query` method returns `TaosResult` class. It provides high level APIs for convenient use: + +Properties: + +- `fields`: the `TaosFields` object contains the column metadata, given the collection of each column field metadata by iterator. +- `field_count`: column number of result. +- `affected_rows`: the rows completed for insert. +- `row_count`: the rows number for select. +- `precision`: the result precision. + +Functions: + +- `fetch_all()`: get all data as tuple array. +- `fetch_all_into_dict()`: get all data as dict array, added since v2.1.1 +- `blocks_iter()`: provides iterator by C `taos_fetch_blocks` API +- `rows_iter()`: provides iterator by C `taos_fetch_row` API +- `fetch_rows_a`: fetch rows by async API in taosc. +- `errno`: error code if failed. +- `errstr`: error string if failed. +- `close`: close result, you do not need to call it directly, result will auto closed out of scope. + - Create subscription ```python @@ -510,6 +576,7 @@ for d in data: sub.close() ``` + - Close connection ```python diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 47bc6686f80496a2d8b51d28783e76842e7336a8..48f0bee6b34496603d67f74938857d7bb94627f2 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -301,3 +301,6 @@ keepColumnName 1 # force TCP transmission # rpcForceTcp 0 + +# unit MB. Flush vnode wal file if walSize > walFlushSize and walSize > cache*0.5*blocks +# walFlushSize 1024 diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 629f8f9fd6a4167db6f30d29646263710602693a..c49bc0a8a356c960e27f3231c3e901de6d9a72ef 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -4,21 +4,19 @@ WORKDIR /root ARG pkgFile ARG dirName -RUN echo ${pkgFile} -RUN echo ${dirName} +RUN echo ${pkgFile} && echo ${dirName} COPY ${pkgFile} /root/ RUN tar -zxf ${pkgFile} WORKDIR /root/${dirName}/ RUN /bin/bash install.sh -e no -RUN apt-get clean && apt-get update && apt-get install -y locales -RUN locale-gen en_US.UTF-8 -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" -ENV LC_CTYPE=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 +RUN apt-get clean && apt-get update && apt-get install -y locales && locale-gen en_US.UTF-8 +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \ + LC_CTYPE=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 CMD ["taosd"] -VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] +VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] \ No newline at end of file diff --git a/packaging/release.sh b/packaging/release.sh index 705103a87a35a73b2a91079707785279416644cd..a827c9ea468277a9c33150447c71f5c93b30d45b 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -213,7 +213,7 @@ else exit 1 fi -make -j8 +make cd ${curr_dir} diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 8015b8dfdc2ccfdc27026575abb6259594b793d9..7fbdbab1c798af572fc67cf79f27812ea64d3bae 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -395,8 +395,9 @@ function install_connector() { ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null else - ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_2_dir}/connector} - ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null || cp ${binary_dir}/build/lib/*.jar ${install_main_2_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || ${csudo} chmod 777 ${install_main_2_dir}/connector/*.jar || echo &> /dev/null + ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_2_dir}/connector + ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null + ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_2_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_2_dir}/connector/*.jar || echo &> /dev/null fi } diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 28515f6c63c98f741d84aa11f92b9ca9f7ad3691..47af7568642d0badccda51a28c09d321cf782571 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: tdengine -base: core18 +base: core20 version: '2.3.0.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. @@ -39,14 +39,17 @@ parts: - taoswrapper.sh tdengine: + plugin: cmake source: . source-type: local - plugin: cmake build-packages: - gcc - g++ - make - cmake + cmake-parameters: + - -DCMAKE_BUILD_TYPE=Release + - -DBUILD_HTTP=true override-build: | snapcraftctl build if [ ! -d $SNAPCRAFT_STAGE/usr ]; then diff --git a/src/client/inc/tscParseLine.h b/src/client/inc/tscParseLine.h index bfc069c92cbfa8c92a889eee3536b4157b4452c3..74ba9ab3d9c5251e1cf8ab4e8549c8da0353ea49 100644 --- a/src/client/inc/tscParseLine.h +++ b/src/client/inc/tscParseLine.h @@ -23,6 +23,8 @@ extern "C" { #define SML_TIMESTAMP_SECOND_DIGITS 10 #define SML_TIMESTAMP_MILLI_SECOND_DIGITS 13 +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + typedef struct { char* key; uint8_t type; @@ -46,31 +48,26 @@ typedef struct { } TAOS_SML_DATA_POINT; typedef enum { - SML_TIME_STAMP_NOW, + SML_TIME_STAMP_NOT_CONFIGURED, SML_TIME_STAMP_HOURS, SML_TIME_STAMP_MINUTES, SML_TIME_STAMP_SECONDS, SML_TIME_STAMP_MILLI_SECONDS, SML_TIME_STAMP_MICRO_SECONDS, SML_TIME_STAMP_NANO_SECONDS, - SML_TIME_STAMP_NOT_CONFIGURED + SML_TIME_STAMP_NOW } SMLTimeStampType; -typedef enum { - SML_LINE_PROTOCOL = 0, - SML_TELNET_PROTOCOL = 1, - SML_JSON_PROTOCOL = 2, -} SMLProtocolType; - typedef struct { uint64_t id; SMLProtocolType protocol; SMLTimeStampType tsType; SHashObj* smlDataToSchema; - int64_t affectedRows; + int32_t affectedRows; } SSmlLinesInfo; +void addEscapeCharToString(char *str, int32_t len); int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info); bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info); bool isValidInteger(char *str); @@ -85,12 +82,12 @@ int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, void destroySmlDataPoint(TAOS_SML_DATA_POINT* point); -int taos_insert_sml_lines(TAOS* taos, char* lines[], int numLines, - SMLProtocolType protocol, SMLTimeStampType tsType); -int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, - SMLProtocolType protocol, SMLTimeStampType tsType); -int taos_insert_json_payload(TAOS* taos, char* payload, - SMLProtocolType protocol, SMLTimeStampType tsType); +int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); +int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); +int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, + SMLTimeStampType tsType, int* affectedRows); #ifdef __cplusplus diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 11ae6ae2704050850e7d79f8ee8c36ce207158e6..e2b151ea6ab417ef18746d0a35ce3c7818b504b7 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -251,6 +251,7 @@ void tscColumnListCopyAll(SArray* dst, const SArray* src); void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo, uint64_t objId, bool convertNchar); void tscDequoteAndTrimToken(SStrToken* pToken); +void tscRmEscapeAndTrimToken(SStrToken* pToken); int32_t tscValidateName(SStrToken* pToken, bool escapeEnabled, bool *dbIncluded); void tscIncStreamExecutionCount(void* pStream); diff --git a/src/client/jni/com_alibaba_datax_plugin_writer_JniConnection.h b/src/client/jni/com_alibaba_datax_plugin_writer_JniConnection.h new file mode 100644 index 0000000000000000000000000000000000000000..61f0e6eb9ce4c15c8c68d8375ad853a7505588ce --- /dev/null +++ b/src/client/jni/com_alibaba_datax_plugin_writer_JniConnection.h @@ -0,0 +1,81 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_alibaba_datax_plugin_writer_JniConnection */ + +#ifndef _Included_com_alibaba_datax_plugin_writer_JniConnection +#define _Included_com_alibaba_datax_plugin_writer_JniConnection +#ifdef __cplusplus +extern "C" { +#endif +#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER +#define com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER 0LL +#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL +#define com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL 0L +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: initImp + * Signature: (Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_initImp + (JNIEnv *, jclass, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: setOptions + * Signature: (ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_setOptions + (JNIEnv *, jclass, jint, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: connectImp + * Signature: (Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_connectImp + (JNIEnv *, jobject, jstring, jint, jstring, jstring, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrCodeImp + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrCodeImp + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrMsgImp + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgImp + (JNIEnv *, jobject, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: freeResultSetImp + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_freeResultSetImp + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: closeConnectionImp + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_closeConnectionImp + (JNIEnv *, jobject, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: insertOpentsdbJson + * Signature: (Ljava/lang/String;J)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_insertOpentsdbJson + (JNIEnv *, jobject, jstring, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/client/jni/jniCommon.h b/src/client/jni/jniCommon.h new file mode 100644 index 0000000000000000000000000000000000000000..78724eed319b8b414b12fc46e3d31899370ba39d --- /dev/null +++ b/src/client/jni/jniCommon.h @@ -0,0 +1,87 @@ +#include + +#ifndef TDENGINE_JNICOMMON_H +#define TDENGINE_JNICOMMON_H + +#define jniFatal(...) \ + { \ + if (jniDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("JNI FATAL ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniError(...) \ + { \ + if (jniDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("JNI ERROR ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniWarn(...) \ + { \ + if (jniDebugFlag & DEBUG_WARN) { \ + taosPrintLog("JNI WARN ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniInfo(...) \ + { \ + if (jniDebugFlag & DEBUG_INFO) { \ + taosPrintLog("JNI ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniDebug(...) \ + { \ + if (jniDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ + } \ + } +#define jniTrace(...) \ + { \ + if (jniDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ + } \ + } + +extern jclass g_arrayListClass; +extern jmethodID g_arrayListConstructFp; +extern jmethodID g_arrayListAddFp; + +extern jclass g_metadataClass; +extern jmethodID g_metadataConstructFp; +extern jfieldID g_metadataColtypeField; +extern jfieldID g_metadataColnameField; +extern jfieldID g_metadataColsizeField; +extern jfieldID g_metadataColindexField; + +extern jclass g_rowdataClass; +extern jmethodID g_rowdataConstructor; +extern jmethodID g_rowdataClearFp; +extern jmethodID g_rowdataSetBooleanFp; +extern jmethodID g_rowdataSetByteFp; +extern jmethodID g_rowdataSetShortFp; +extern jmethodID g_rowdataSetIntFp; +extern jmethodID g_rowdataSetLongFp; +extern jmethodID g_rowdataSetFloatFp; +extern jmethodID g_rowdataSetDoubleFp; +extern jmethodID g_rowdataSetStringFp; +extern jmethodID g_rowdataSetTimestampFp; +extern jmethodID g_rowdataSetByteArrayFp; + +extern jmethodID g_blockdataSetByteArrayFp; +extern jmethodID g_blockdataSetNumOfRowsFp; +extern jmethodID g_blockdataSetNumOfColsFp; + +#define JNI_SUCCESS 0 +#define JNI_TDENGINE_ERROR -1 +#define JNI_CONNECTION_NULL -2 +#define JNI_RESULT_SET_NULL -3 +#define JNI_NUM_OF_FIELDS_0 -4 +#define JNI_SQL_NULL -5 +#define JNI_FETCH_END -6 +#define JNI_OUT_OF_MEMORY -7 + +extern JavaVM *g_vm; + +void jniGetGlobalMethod(JNIEnv *env); + +int32_t check_for_params(jobject jobj, jlong conn, jlong res); + +#endif // TDENGINE_JNICOMMON_H diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 0444c2cb8dd23f2e73179668e6cb7195a030b6be..ef7387f9760b7d710f77a97b52fcafe9686bd335 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -17,46 +17,9 @@ #include "taos.h" #include "tlog.h" #include "tscUtil.h" -#include "tscParseLine.h" #include "com_taosdata_jdbc_TSDBJNIConnector.h" - -#define jniFatal(...) \ - { \ - if (jniDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("JNI FATAL ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniError(...) \ - { \ - if (jniDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("JNI ERROR ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniWarn(...) \ - { \ - if (jniDebugFlag & DEBUG_WARN) { \ - taosPrintLog("JNI WARN ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniInfo(...) \ - { \ - if (jniDebugFlag & DEBUG_INFO) { \ - taosPrintLog("JNI ", tscEmbedded ? 255 : jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniDebug(...) \ - { \ - if (jniDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ - } \ - } -#define jniTrace(...) \ - { \ - if (jniDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); \ - } \ - } +#include "jniCommon.h" int __init = 0; @@ -91,16 +54,7 @@ jmethodID g_blockdataSetByteArrayFp; jmethodID g_blockdataSetNumOfRowsFp; jmethodID g_blockdataSetNumOfColsFp; -#define JNI_SUCCESS 0 -#define JNI_TDENGINE_ERROR -1 -#define JNI_CONNECTION_NULL -2 -#define JNI_RESULT_SET_NULL -3 -#define JNI_NUM_OF_FIELDS_0 -4 -#define JNI_SQL_NULL -5 -#define JNI_FETCH_END -6 -#define JNI_OUT_OF_MEMORY -7 - -static void jniGetGlobalMethod(JNIEnv *env) { +void jniGetGlobalMethod(JNIEnv *env) { // make sure init function executed once switch (atomic_val_compare_exchange_32(&__init, 0, 1)) { case 0: @@ -159,7 +113,7 @@ static void jniGetGlobalMethod(JNIEnv *env) { jniDebug("native method register finished"); } -static int32_t check_for_params(jobject jobj, jlong conn, jlong res) { +int32_t check_for_params(jobject jobj, jlong conn, jlong res) { if ((TAOS *)conn == NULL) { jniError("jobj:%p, connection is closed", jobj); return JNI_CONNECTION_NULL; @@ -219,26 +173,8 @@ JNIEXPORT jobject createTSDBException(JNIEnv *env, int code, char *msg) { return exception_obj; } -/* - * Class: com_taosdata_jdbc_TSDBJNIConnector - * Method: setConfigImp - * Signature: (Ljava/lang/String;)Lcom/taosdata/jdbc/TSDBException; - */ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setConfigImp(JNIEnv *env, jclass jobj, jstring config) { - /* - if (config == NULL) { - jniDebug("config value is null"); - return -1; - } - - const char *cfg = (*env)->GetStringUTFChars(env, config, NULL); - if (!cfg) { - return -1; - } - return 0; - */ - if (config == NULL) { char *msg = "config value is null"; jniDebug("config value is null"); @@ -254,7 +190,7 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setConfigImp(J setConfRet result = taos_set_config(cfg); int code = result.retCode; - char * msg = result.retMsg; + char *msg = result.retMsg; return createTSDBException(env, code, msg); } @@ -424,7 +360,7 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp(J JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp(JNIEnv *env, jobject jobj, jlong con, jlong tres) { - TAOS * tscon = (TAOS *)con; + TAOS *tscon = (TAOS *)con; int32_t code = check_for_params(jobj, con, tres); if (code != JNI_SUCCESS) { return code; @@ -467,7 +403,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp( JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsImp(JNIEnv *env, jobject jobj, jlong con, jlong res) { - TAOS * tscon = (TAOS *)con; + TAOS *tscon = (TAOS *)con; int32_t code = check_for_params(jobj, con, res); if (code != JNI_SUCCESS) { return code; @@ -483,13 +419,13 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getAffectedRowsIm JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaDataImp(JNIEnv *env, jobject jobj, jlong con, jlong res, jobject arrayListObj) { - TAOS * tscon = (TAOS *)con; + TAOS *tscon = (TAOS *)con; int32_t code = check_for_params(jobj, con, res); if (code != JNI_SUCCESS) { return code; } - TAOS_RES * tres = (TAOS_RES *)res; + TAOS_RES *tres = (TAOS_RES *)res; TAOS_FIELD *fields = taos_fetch_fields(tres); int32_t num_fields = taos_num_fields(tres); @@ -626,13 +562,13 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNIEnv *env, jobject jobj, jlong con, jlong res, jobject rowobj) { - TAOS * tscon = (TAOS *)con; + TAOS *tscon = (TAOS *)con; int32_t code = check_for_params(jobj, con, res); if (code != JNI_SUCCESS) { return code; } - TAOS_RES * tres = (TAOS_RES *)res; + TAOS_RES *tres = (TAOS_RES *)res; TAOS_FIELD *fields = taos_fetch_fields(tres); int32_t numOfFields = taos_num_fields(tres); @@ -1021,7 +957,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI } const char *name = (*env)->GetStringUTFChars(env, tableName, NULL); - char * curTags = tagsData; + char *curTags = tagsData; TAOS_BIND *tagsBind = calloc(numOfTags, sizeof(TAOS_BIND)); for (int32_t i = 0; i < numOfTags; ++i) { @@ -1053,7 +989,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI } JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(JNIEnv *env, jobject jobj, - jobjectArray lines, jlong conn) { + jobjectArray lines, jlong conn, + jint protocol, jint precision) { TAOS *taos = (TAOS *)conn; if (taos == NULL) { jniError("jobj:%p, connection already closed", jobj); @@ -1071,7 +1008,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(J c_lines[i] = (char *)(*env)->GetStringUTFChars(env, line, 0); } - int code = taos_schemaless_insert(taos, c_lines, numLines, SML_LINE_PROTOCOL, "ms"); + SSqlObj* result = (SSqlObj*)taos_schemaless_insert(taos, c_lines, numLines, protocol, precision); + int code = taos_errno(result); for (int i = 0; i < numLines; ++i) { jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); @@ -1080,9 +1018,10 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(J tfree(c_lines); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s", jobj, taos, tstrerror(code)); + jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, taos, tstrerror(code), taos_errstr(result)); return JNI_TDENGINE_ERROR; } - return code; + + return (jlong)result; } diff --git a/src/client/src/dataxJniConnection.c b/src/client/src/dataxJniConnection.c new file mode 100644 index 0000000000000000000000000000000000000000..beaa61dac16d790cb01ca616140801fade46eee1 --- /dev/null +++ b/src/client/src/dataxJniConnection.c @@ -0,0 +1,232 @@ +#include "os.h" +#include "taos.h" +#include "tlog.h" +#include "tscUtil.h" + +#include "com_alibaba_datax_plugin_writer_JniConnection.h" +#include "jniCommon.h" + +jclass g_arrayListClass; +jmethodID g_arrayListConstructFp; +jmethodID g_arrayListAddFp; + +jclass g_metadataClass; +jmethodID g_metadataConstructFp; +jfieldID g_metadataColtypeField; +jfieldID g_metadataColnameField; +jfieldID g_metadataColsizeField; +jfieldID g_metadataColindexField; + +jclass g_rowdataClass; +jmethodID g_rowdataConstructor; +jmethodID g_rowdataClearFp; +jmethodID g_rowdataSetBooleanFp; +jmethodID g_rowdataSetByteFp; +jmethodID g_rowdataSetShortFp; +jmethodID g_rowdataSetIntFp; +jmethodID g_rowdataSetLongFp; +jmethodID g_rowdataSetFloatFp; +jmethodID g_rowdataSetDoubleFp; +jmethodID g_rowdataSetStringFp; +jmethodID g_rowdataSetTimestampFp; +jmethodID g_rowdataSetByteArrayFp; + +jmethodID g_blockdataSetByteArrayFp; +jmethodID g_blockdataSetNumOfRowsFp; +jmethodID g_blockdataSetNumOfColsFp; + +JNIEXPORT void JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_initImp(JNIEnv *env, jobject jobj, + jstring jconfigDir) { + if (jconfigDir != NULL) { + const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL); + if (confDir && strlen(confDir) != 0) { + tstrncpy(configDir, confDir, TSDB_FILENAME_LEN); + } + (*env)->ReleaseStringUTFChars(env, jconfigDir, confDir); + } + + jniDebug("jni initialized successfully, config directory: %s", configDir); +} + +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_setOptions(JNIEnv *env, jobject jobj, + jint optionIndex, + jstring optionValue) { + if (optionValue == NULL) { + jniDebug("option index:%d value is null", (int32_t)optionIndex); + return 0; + } + + int res = 0; + + if (optionIndex == TSDB_OPTION_LOCALE) { + const char *locale = (*env)->GetStringUTFChars(env, optionValue, NULL); + if (locale && strlen(locale) != 0) { + res = taos_options(TSDB_OPTION_LOCALE, locale); + jniDebug("set locale to %s, result:%d", locale, res); + } else { + jniDebug("input locale is empty"); + } + (*env)->ReleaseStringUTFChars(env, optionValue, locale); + } else if (optionIndex == TSDB_OPTION_CHARSET) { + const char *charset = (*env)->GetStringUTFChars(env, optionValue, NULL); + if (charset && strlen(charset) != 0) { + res = taos_options(TSDB_OPTION_CHARSET, charset); + jniDebug("set character encoding to %s, result:%d", charset, res); + } else { + jniDebug("input character encoding is empty"); + } + (*env)->ReleaseStringUTFChars(env, optionValue, charset); + } else if (optionIndex == TSDB_OPTION_TIMEZONE) { + const char *tz1 = (*env)->GetStringUTFChars(env, optionValue, NULL); + if (tz1 && strlen(tz1) != 0) { + res = taos_options(TSDB_OPTION_TIMEZONE, tz1); + jniDebug("set timezone to %s, result:%d", tz1, res); + } else { + jniDebug("input timezone is empty"); + } + (*env)->ReleaseStringUTFChars(env, optionValue, tz1); + } else { + jniError("option index:%d is not found", (int32_t)optionIndex); + } + + return res; +} + +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_connectImp(JNIEnv *env, jobject jobj, + jstring jhost, jint jport, + jstring jdbName, jstring juser, + jstring jpass) { + jlong ret = 0; + const char *host = NULL; + const char *user = NULL; + const char *pass = NULL; + const char *dbname = NULL; + + if (jhost != NULL) { + host = (*env)->GetStringUTFChars(env, jhost, NULL); + } + + if (jdbName != NULL) { + dbname = (*env)->GetStringUTFChars(env, jdbName, NULL); + } + + if (juser != NULL) { + user = (*env)->GetStringUTFChars(env, juser, NULL); + } + + if (jpass != NULL) { + pass = (*env)->GetStringUTFChars(env, jpass, NULL); + } + + if (user == NULL) { + jniDebug("jobj:%p, user not specified, use default user %s", jobj, TSDB_DEFAULT_USER); + } + + if (pass == NULL) { + jniDebug("jobj:%p, pass not specified, use default password", jobj); + } + + ret = (jlong)taos_connect((char *)host, (char *)user, (char *)pass, (char *)dbname, (uint16_t)jport); + if (ret == 0) { + jniError("jobj:%p, conn:%p, connect to database failed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, + (char *)host, (char *)user, (char *)dbname, (int32_t)jport); + } else { + jniDebug("jobj:%p, conn:%p, connect to database succeed, host=%s, user=%s, dbname=%s, port=%d", jobj, (void *)ret, + (char *)host, (char *)user, (char *)dbname, (int32_t)jport); + } + + if (host != NULL) { + (*env)->ReleaseStringUTFChars(env, jhost, host); + } + + if (dbname != NULL) { + (*env)->ReleaseStringUTFChars(env, jdbName, dbname); + } + + if (user != NULL) { + (*env)->ReleaseStringUTFChars(env, juser, user); + } + + if (pass != NULL) { + (*env)->ReleaseStringUTFChars(env, jpass, pass); + } + + return ret; +} + +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrCodeImp(JNIEnv *env, jobject jobj, + jlong con, jlong tres) { + int32_t code = check_for_params(jobj, con, tres); + if (code != JNI_SUCCESS) { + return code; + } + + return (jint)taos_errno((TAOS_RES *)tres); +} + +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgImp(JNIEnv *env, jobject jobj, + jlong tres) { + TAOS_RES *pSql = (TAOS_RES *)tres; + return (*env)->NewStringUTF(env, (const char *)taos_errstr(pSql)); +} + +JNIEXPORT void JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_freeResultSetImp(JNIEnv *env, jobject jobj, + jlong con, jlong res) { + if ((TAOS *)con == NULL) { + jniError("jobj:%p, connection is closed", jobj); + } + if ((TAOS_RES *)res == NULL) { + jniError("jobj:%p, conn:%p, res is null", jobj, (TAOS *)con); + } + taos_free_result((TAOS_RES *)res); + jniDebug("jobj:%p, conn:%p, free resultset:%p", jobj, (TAOS *)con, (void *)res); +} + +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_closeConnectionImp(JNIEnv *env, jobject jobj, + jlong con) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection is already closed", jobj); + return JNI_CONNECTION_NULL; + } else { + jniDebug("jobj:%p, conn:%p, close connection success", jobj, tscon); + taos_close(tscon); + return JNI_SUCCESS; + } +} + +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_insertOpentsdbJson(JNIEnv *env, jobject jobj, + jstring json, jlong con) { + // check connection + TAOS *conn = (TAOS *)con; + if (conn == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + // java.lang.String -> char * + char *payload = NULL; + if (json != NULL) { + payload = (char *)(*env)->GetStringUTFChars(env, json, NULL); + } + // check payload + if (payload == NULL) { + jniDebug("jobj:%p, invalid argument: opentsdb insert json is NULL", jobj); + return JNI_SQL_NULL; + } + // schemaless insert + char *payload_arr[1]; + payload_arr[0] = payload; + TAOS_RES *result; + result = taos_schemaless_insert(conn, payload_arr, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + + int code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, conn, tstrerror(code), taos_errstr(result)); + } else { + int32_t affectRows = taos_affected_rows(result); + jniDebug("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, conn, tstrerror(code), affectRows); + } + + (*env)->ReleaseStringUTFChars(env, json, payload); + return (jlong)result; +} \ No newline at end of file diff --git a/src/client/src/taos.def b/src/client/src/taos.def index 28a9dde2239435b1b916e00fe05ca5634e7bbcfc..0e7289764b28d6b40d6576afb125f4251e88182f 100644 --- a/src/client/src/taos.def +++ b/src/client/src/taos.def @@ -51,3 +51,4 @@ taos_stmt_bind_param_batch taos_stmt_bind_single_param_batch taos_is_null taos_insert_lines +taos_schemaless_insert diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index da03ed40f02c667c474f3c10a648cb1808667835..dbc7503535d79604c3ca908ce748075d41e2eb00 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1251,10 +1251,18 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat sToken = tStrGetToken(str, &index, false); str += index; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character backstick(`) + strncpy(tmpTokenBuf, sToken.z, sToken.n); + sToken.z = tmpTokenBuf; + if (TK_STRING == sToken.type) { tscDequoteAndTrimToken(&sToken); } + if (TK_ID == sToken.type) { + tscRmEscapeAndTrimToken(&sToken); + } + if (sToken.type == TK_RP) { if (end != NULL) { // set the end position *end = str; diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index f63a19452f1fc64483e32b9eb0f5c24b1b5f1d14..64b15d0dd85c5acc83eee9d18fe191817383d9ef 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -499,6 +499,7 @@ static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSc for (int i=0; itableInfo.numOfColumns; ++i) { SSchema field; tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); + addEscapeCharToString(field.name, (int16_t)strlen(field.name)); field.type = tableMeta->schema[i].type; field.bytes = tableMeta->schema[i].bytes; taosArrayPush(schema->fields, &field); @@ -510,6 +511,7 @@ static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSc int j = i + tableMeta->tableInfo.numOfColumns; SSchema field; tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); + addEscapeCharToString(field.name, (int16_t)strlen(field.name)); field.type = tableMeta->schema[j].type; field.bytes = tableMeta->schema[j].bytes; taosArrayPush(schema->tags, &field); @@ -1175,6 +1177,15 @@ static void escapeSpecialCharacter(uint8_t field, const char **pos) { *pos = cur; } +void addEscapeCharToString(char *str, int32_t len) { + if (str == NULL) { + return; + } + memmove(str + 1, str, len); + str[0] = str[len + 1] = TS_ESCAPE_CHAR; + str[len + 2] = '\0'; +} + bool isValidInteger(char *str) { char *c = str; if (*c != '+' && *c != '-' && !isdigit(*c)) { @@ -1435,58 +1446,65 @@ static bool isNchar(char *pVal, uint16_t len) { return false; } -static bool isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType, SSmlLinesInfo* info) { +static int32_t isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType, SSmlLinesInfo* info) { if (len == 0) { - return true; + return TSDB_CODE_SUCCESS; } if ((len == 1) && pVal[0] == '0') { *tsType = SML_TIME_STAMP_NOW; - return true; + return TSDB_CODE_SUCCESS; } - //Default no appendix - if (isdigit(pVal[len - 1]) && isdigit(pVal[len - 2])) { - if (info->protocol == SML_LINE_PROTOCOL) { - if (info->tsType != SML_TIME_STAMP_NOT_CONFIGURED) { - *tsType = info->tsType; - } else { - *tsType = SML_TIME_STAMP_NANO_SECONDS; - } - } else if (info->protocol == SML_TELNET_PROTOCOL) { - if (len == SML_TIMESTAMP_SECOND_DIGITS) { - *tsType = SML_TIME_STAMP_SECONDS; - } else if (len == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - } else { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } + for (int i = 0; i < len; ++i) { + if(!isdigit(pVal[i])) { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - return true; } - if (pVal[len - 1] == 's') { - switch (pVal[len - 2]) { - case 'm': - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - break; - case 'u': - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - break; - case 'n': - *tsType = SML_TIME_STAMP_NANO_SECONDS; - break; - default: - if (isdigit(pVal[len - 2])) { - *tsType = SML_TIME_STAMP_SECONDS; - break; - } else { - return false; - } + /* For InfluxDB line protocol use user passed timestamp precision + * For OpenTSDB protocols only 10 digit(seconds) or 13 digits(milliseconds) + * precision allowed + */ + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { + if (info->tsType != SML_TIME_STAMP_NOT_CONFIGURED) { + *tsType = info->tsType; + } else { + *tsType = SML_TIME_STAMP_NANO_SECONDS; + } + } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { + if (len == SML_TIMESTAMP_SECOND_DIGITS) { + *tsType = SML_TIME_STAMP_SECONDS; + } else if (len == SML_TIMESTAMP_MILLI_SECOND_DIGITS) { + *tsType = SML_TIME_STAMP_MILLI_SECONDS; + } else { + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - //printf("Type is timestamp(%s)\n", pVal); - return true; } - return false; + return TSDB_CODE_SUCCESS; + + //if (pVal[len - 1] == 's') { + // switch (pVal[len - 2]) { + // case 'm': + // *tsType = SML_TIME_STAMP_MILLI_SECONDS; + // break; + // case 'u': + // *tsType = SML_TIME_STAMP_MICRO_SECONDS; + // break; + // case 'n': + // *tsType = SML_TIME_STAMP_NANO_SECONDS; + // break; + // default: + // if (isdigit(pVal[len - 2])) { + // *tsType = SML_TIME_STAMP_SECONDS; + // break; + // } else { + // return false; + // } + // } + // //printf("Type is timestamp(%s)\n", pVal); + // return true; + //} + //return false; } static bool convertStrToNumber(TAOS_SML_KV *pVal, char *str, SSmlLinesInfo* info) { @@ -1750,14 +1768,6 @@ bool convertSmlValueType(TAOS_SML_KV *pVal, char *value, static int32_t getTimeStampValue(char *value, uint16_t len, SMLTimeStampType type, int64_t *ts, SSmlLinesInfo* info) { - if (len >= 2) { - for (int i = 0; i < len - 2; ++i) { - if(!isdigit(value[i])) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; - } - } - } - //No appendix or no timestamp given (len = 0) if (len != 0 && type != SML_TIME_STAMP_NOW) { *ts = (int64_t)strtoll(value, NULL, 10); @@ -1806,13 +1816,13 @@ int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value, SMLTimeStampType type; int64_t tsVal; - strntolower_s(value, value, len); - if (!isTimeStamp(value, len, &type, info)) { - return TSDB_CODE_TSC_INVALID_TIME_STAMP; + ret = isTimeStamp(value, len, &type, info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } ret = getTimeStampValue(value, len, type, &tsVal, info); - if (ret) { + if (ret != TSDB_CODE_SUCCESS) { return ret; } tscDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal); @@ -1884,15 +1894,10 @@ bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) { static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) { const char *cur = *index; char key[TSDB_COL_NAME_LEN + 1]; // +1 to avoid key[len] over write - uint16_t len = 0; + int16_t len = 0; - //key field cannot start with digit - if (isdigit(*cur)) { - tscError("SML:0x%"PRIx64" Tag key cannot start with digit", info->id); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } while (*cur != '\0') { - if (len >= TSDB_COL_NAME_LEN - 1) { + if (len > TSDB_COL_NAME_LEN - 1) { tscError("SML:0x%"PRIx64" Key field cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -1919,9 +1924,11 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; } - pKV->key = calloc(len + 1, 1); + pKV->key = calloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1); memcpy(pKV->key, key, len + 1); - //tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); + strntolower_s(pKV->key, pKV->key, (int32_t)len); + addEscapeCharToString(pKV->key, len); + tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); *index = cur + 1; return TSDB_CODE_SUCCESS; } @@ -1932,7 +1939,7 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **index, const char *start, *cur; int32_t ret = TSDB_CODE_SUCCESS; char *value = NULL; - uint16_t len = 0; + int16_t len = 0; bool searchQuote = false; start = cur = *index; @@ -2013,21 +2020,15 @@ error: static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index, uint8_t *has_tags, SSmlLinesInfo* info) { const char *cur = *index; - uint16_t len = 0; + int16_t len = 0; - pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + 1, 1); // +1 to avoid 1772 line over write + pSml->stableName = calloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1); if (pSml->stableName == NULL){ return TSDB_CODE_TSC_OUT_OF_MEMORY; } - if (isdigit(*cur)) { - tscError("SML:0x%"PRIx64" Measurement field cannnot start with digit", info->id); - free(pSml->stableName); - pSml->stableName = NULL; - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } while (*cur != '\0') { - if (len >= TSDB_TABLE_NAME_LEN - 1) { + if (len > TSDB_TABLE_NAME_LEN - 1) { tscError("SML:0x%"PRIx64" Measurement field cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); free(pSml->stableName); pSml->stableName = NULL; @@ -2061,7 +2062,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index pSml->stableName = NULL; return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; } - pSml->stableName[len] = '\0'; + addEscapeCharToString(pSml->stableName, len); *index = cur + 1; tscDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len); @@ -2117,17 +2118,11 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs, tscError("SML:0x%"PRIx64" Unable to parse value", info->id); goto error; } - if (!isField && (strcasecmp(pkv->key, "ID") == 0)) { - ret = isValidChildTableName(pkv->value, pkv->length, info); - if (ret) { - free(pkv->key); - free(pkv->value); - goto error; - } - smlData->childTableName = malloc( pkv->length + 1); + if (!isField && (strcasecmp(pkv->key, "`ID`") == 0)) { + smlData->childTableName = malloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1); memcpy(smlData->childTableName, pkv->value, pkv->length); strntolower_s(smlData->childTableName, smlData->childTableName, (int32_t)pkv->length); - smlData->childTableName[pkv->length] = '\0'; + addEscapeCharToString(smlData->childTableName, (int32_t)pkv->length); free(pkv->key); free(pkv->value); } else { @@ -2273,7 +2268,7 @@ int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* faile return TSDB_CODE_SUCCESS; } -int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType) { +int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int *affectedRows) { int32_t code = 0; SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); @@ -2317,6 +2312,9 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType p if (code != 0) { tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); } + if (affectedRows != NULL) { + *affectedRows = info->affectedRows; + } cleanup: tscDebug("SML:0x%"PRIx64" taos_insert_lines finish inserting %d lines. code: %d", info->id, numLines, code); @@ -2332,52 +2330,57 @@ cleanup: return code; } -int32_t convertPrecisionStrType(char* precision, SMLTimeStampType *tsType) { - if (precision == NULL) { - *tsType = SML_TIME_STAMP_NOT_CONFIGURED; - return TSDB_CODE_SUCCESS; - } - if (strcmp(precision, "μ") == 0) { - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - return TSDB_CODE_SUCCESS; +static int32_t convertPrecisionType(int precision, SMLTimeStampType *tsType) { + switch (precision) { + case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: + *tsType = SML_TIME_STAMP_NOT_CONFIGURED; + break; + case TSDB_SML_TIMESTAMP_HOURS: + *tsType = SML_TIME_STAMP_HOURS; + break; + case TSDB_SML_TIMESTAMP_MILLI_SECONDS: + *tsType = SML_TIME_STAMP_MILLI_SECONDS; + break; + case TSDB_SML_TIMESTAMP_NANO_SECONDS: + *tsType = SML_TIME_STAMP_NANO_SECONDS; + break; + case TSDB_SML_TIMESTAMP_MICRO_SECONDS: + *tsType = SML_TIME_STAMP_MICRO_SECONDS; + break; + case TSDB_SML_TIMESTAMP_SECONDS: + *tsType = SML_TIME_STAMP_SECONDS; + break; + case TSDB_SML_TIMESTAMP_MINUTES: + *tsType = SML_TIME_STAMP_MINUTES; + break; + default: + return TSDB_CODE_TSC_INVALID_PRECISION_TYPE; } - int32_t len = (int32_t)strlen(precision); - if (len == 1) { - switch (precision[0]) { - case 'u': - *tsType = SML_TIME_STAMP_MICRO_SECONDS; - break; - case 's': - *tsType = SML_TIME_STAMP_SECONDS; - break; - case 'm': - *tsType = SML_TIME_STAMP_MINUTES; - break; - case 'h': - *tsType = SML_TIME_STAMP_HOURS; - break; - default: - return TSDB_CODE_TSC_INVALID_PRECISION_TYPE; - } - } else if (len == 2 && precision[1] == 's') { - switch (precision[0]) { - case 'm': - *tsType = SML_TIME_STAMP_MILLI_SECONDS; - break; - case 'n': - *tsType = SML_TIME_STAMP_NANO_SECONDS; - break; - default: - return TSDB_CODE_TSC_INVALID_PRECISION_TYPE; - } - } else { - return TSDB_CODE_TSC_INVALID_PRECISION_TYPE; + return TSDB_CODE_SUCCESS; +} + +//make a dummy SSqlObj +static SSqlObj* createSmlQueryObj(TAOS* taos, int32_t affected_rows, int32_t code) { + SSqlObj *pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); + if (pNew == NULL) { + return NULL; } + pNew->signature = pNew; + pNew->pTscObj = taos; + pNew->fp = NULL; - return TSDB_CODE_SUCCESS; + tsem_init(&pNew->rspSem, 0, 0); + registerSqlObj(pNew); + + pNew->res.numOfRows = affected_rows; + pNew->res.code = code; + + + return pNew; } + /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -2399,31 +2402,35 @@ int32_t convertPrecisionStrType(char* precision, SMLTimeStampType *tsType) { * */ -int taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, char* timePrecision) { - int code; +TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { + int code = TSDB_CODE_SUCCESS; + int affected_rows = 0; SMLTimeStampType tsType; - if (protocol == SML_LINE_PROTOCOL) { - code = convertPrecisionStrType(timePrecision, &tsType); + if (protocol == TSDB_SML_LINE_PROTOCOL) { + code = convertPrecisionType(precision, &tsType); if (code != TSDB_CODE_SUCCESS) { - return code; + return NULL; } } switch (protocol) { - case SML_LINE_PROTOCOL: - code = taos_insert_lines(taos, lines, numLines, protocol, tsType); + case TSDB_SML_LINE_PROTOCOL: + code = taos_insert_lines(taos, lines, numLines, protocol, tsType, &affected_rows); break; - case SML_TELNET_PROTOCOL: - code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType); + case TSDB_SML_TELNET_PROTOCOL: + code = taos_insert_telnet_lines(taos, lines, numLines, protocol, tsType, &affected_rows); break; - case SML_JSON_PROTOCOL: - code = taos_insert_json_payload(taos, *lines, protocol, tsType); + case TSDB_SML_JSON_PROTOCOL: + code = taos_insert_json_payload(taos, *lines, protocol, tsType, &affected_rows); break; default: code = TSDB_CODE_TSC_INVALID_PROTOCOL_TYPE; break; } - return code; + + SSqlObj *pSql = createSmlQueryObj(taos, affected_rows, code); + + return (TAOS_RES*)pSql; } diff --git a/src/client/src/tscParseOpenTSDB.c b/src/client/src/tscParseOpenTSDB.c index a6c1acfc35903750e412775b53be924fc5550b87..f6b723ef3cd554a4062035c6352ee485022340ac 100644 --- a/src/client/src/tscParseOpenTSDB.c +++ b/src/client/src/tscParseOpenTSDB.c @@ -37,18 +37,20 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, const char *cur = *index; uint16_t len = 0; - pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN, 1); + pSml->stableName = tcalloc(TSDB_TABLE_NAME_LEN + TS_ESCAPE_CHAR_SIZE, 1); if (pSml->stableName == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } + /* if (isdigit(*cur)) { tscError("OTD:0x%"PRIx64" Metric cannot start with digit", info->id); tfree(pSml->stableName); return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; } + */ while (*cur != '\0') { - if (len >= TSDB_TABLE_NAME_LEN - 1) { + if (len > TSDB_TABLE_NAME_LEN - 1) { tscError("OTD:0x%"PRIx64" Metric cannot exceeds %d characters", info->id, TSDB_TABLE_NAME_LEN - 1); tfree(pSml->stableName); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; @@ -63,7 +65,7 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, } } - pSml->stableName[len] = *cur; + pSml->stableName[len] = tolower(*cur); cur++; len++; @@ -73,7 +75,7 @@ static int32_t parseTelnetMetric(TAOS_SML_DATA_POINT *pSml, const char **index, return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; } - pSml->stableName[len] = '\0'; + addEscapeCharToString(pSml->stableName, len); *index = cur + 1; tscDebug("OTD:0x%"PRIx64" Stable name in metric:%s|len:%d", info->id, pSml->stableName, len); @@ -207,12 +209,12 @@ static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj uint16_t len = 0; //key field cannot start with digit - if (isdigit(*cur)) { - tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id); - return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; - } + //if (isdigit(*cur)) { + // tscError("OTD:0x%"PRIx64" Tag key cannot start with digit", info->id); + // return TSDB_CODE_TSC_LINE_SYNTAX_ERROR; + //} while (*cur != '\0') { - if (len >= TSDB_COL_NAME_LEN - 1) { + if (len > TSDB_COL_NAME_LEN - 1) { tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters", info->id, TSDB_COL_NAME_LEN - 1); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } @@ -236,8 +238,10 @@ static int32_t parseTelnetTagKey(TAOS_SML_KV *pKV, const char **index, SHashObj return TSDB_CODE_TSC_DUP_TAG_NAMES; } - pKV->key = tcalloc(len + 1, 1); + pKV->key = tcalloc(len + TS_ESCAPE_CHAR_SIZE + 1, 1); memcpy(pKV->key, key, len + 1); + strntolower_s(pKV->key, pKV->key, (int32_t)len); + addEscapeCharToString(pKV->key, len); //tscDebug("OTD:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len); *index = cur + 1; return TSDB_CODE_SUCCESS; @@ -312,15 +316,12 @@ static int32_t parseTelnetTagKvs(TAOS_SML_KV **pKVs, int *num_kvs, tscError("OTD:0x%"PRIx64" Unable to parse value", info->id); return ret; } - if ((strcasecmp(pkv->key, "ID") == 0)) { - ret = isValidChildTableName(pkv->value, pkv->length, info); - if (ret) { - return ret; - } - *childTableName = malloc(pkv->length + 1); + if ((strcasecmp(pkv->key, "`ID`") == 0)) { + *childTableName = tcalloc(pkv->length + TS_ESCAPE_CHAR_SIZE + 1, 1); memcpy(*childTableName, pkv->value, pkv->length); (*childTableName)[pkv->length] = '\0'; strntolower_s(*childTableName, *childTableName, (int32_t)pkv->length); + addEscapeCharToString(*childTableName, pkv->length); tfree(pkv->key); tfree(pkv->value); } else { @@ -409,7 +410,7 @@ static int32_t tscParseTelnetLines(char* lines[], int numLines, SArray* points, return TSDB_CODE_SUCCESS; } -int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType) { +int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { int32_t code = 0; SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); @@ -453,6 +454,9 @@ int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines, SMLProtoco if (code != 0) { tscError("OTD:0x%"PRIx64" taos_insert_telnet_lines error: %s", info->id, tstrerror((code))); } + if (affectedRows != NULL) { + *affectedRows = info->affectedRows; + } cleanup: tscDebug("OTD:0x%"PRIx64" taos_insert_telnet_lines finish inserting %d lines. code: %d", info->id, numLines, code); @@ -490,19 +494,22 @@ static int32_t parseMetricFromJSON(cJSON *root, TAOS_SML_DATA_POINT* pSml, SSmlL return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; } - pSml->stableName = tcalloc(stableLen + 1, sizeof(char)); + pSml->stableName = tcalloc(stableLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char)); if (pSml->stableName == NULL){ return TSDB_CODE_TSC_OUT_OF_MEMORY; } + /* if (isdigit(metric->valuestring[0])) { tscError("OTD:0x%"PRIx64" Metric cannot start with digit in JSON", info->id); tfree(pSml->stableName); return TSDB_CODE_TSC_INVALID_JSON; } + */ tstrncpy(pSml->stableName, metric->valuestring, stableLen + 1); strntolower_s(pSml->stableName, pSml->stableName, (int32_t)stableLen); + addEscapeCharToString(pSml->stableName, (int32_t)stableLen); return TSDB_CODE_SUCCESS; @@ -885,7 +892,6 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, if (tags == NULL || tags->type != cJSON_Object) { return TSDB_CODE_TSC_INVALID_JSON; } - //only pick up the first ID value as child table name cJSON *id = cJSON_GetObjectItem(tags, "ID"); if (id != NULL) { @@ -894,13 +900,10 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, return TSDB_CODE_TSC_INVALID_JSON; } size_t idLen = strlen(id->valuestring); - ret = isValidChildTableName(id->valuestring, (int16_t)idLen, info); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - *childTableName = tcalloc(idLen + 1, sizeof(char)); + *childTableName = tcalloc(idLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char)); memcpy(*childTableName, id->valuestring, idLen); strntolower_s(*childTableName, *childTableName, (int32_t)idLen); + addEscapeCharToString(*childTableName, (int32_t)idLen); //check duplicate IDs cJSON_DeleteItemFromObject(tags, "ID"); @@ -909,7 +912,6 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, return TSDB_CODE_TSC_DUP_TAG_NAMES; } } - int32_t tagNum = cJSON_GetArraySize(tags); //at least one tag pair required if (tagNum <= 0) { @@ -935,8 +937,10 @@ static int32_t parseTagsFromJSON(cJSON *root, TAOS_SML_KV **pKVs, int *num_kvs, tscError("OTD:0x%"PRIx64" Tag key cannot exceeds %d characters in JSON", info->id, TSDB_COL_NAME_LEN - 1); return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; } - pkv->key = tcalloc(keyLen + 1, sizeof(char)); + pkv->key = tcalloc(keyLen + TS_ESCAPE_CHAR_SIZE + 1, sizeof(char)); strncpy(pkv->key, tag->string, keyLen); + strntolower_s(pkv->key, pkv->key, (int32_t)keyLen); + addEscapeCharToString(pkv->key, (int32_t)keyLen); //value ret = parseValueFromJSON(tag, pkv, info); if (ret != TSDB_CODE_SUCCESS) { @@ -1045,7 +1049,7 @@ PARSE_JSON_OVER: return ret; } -int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType) { +int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol, SMLTimeStampType tsType, int* affectedRows) { int32_t code = 0; SSmlLinesInfo* info = tcalloc(1, sizeof(SSmlLinesInfo)); @@ -1080,6 +1084,9 @@ int taos_insert_json_payload(TAOS* taos, char* payload, SMLProtocolType protocol if (code != 0) { tscError("OTD:0x%"PRIx64" taos_insert_json_payload error: %s", info->id, tstrerror((code))); } + if (affectedRows != NULL) { + *affectedRows = info->affectedRows; + } cleanup: tscDebug("OTD:0x%"PRIx64" taos_insert_json_payload finish inserting 1 Point. code: %d", info->id, code); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ed6c81bff6d277615ba45b453c65182ac6a03d82..e96e7e5b232a500f5780688cab8a82ab6545e624 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3112,10 +3112,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col memset(pExpr->base.aliasName, 0, tListLen(pExpr->base.aliasName)); getColumnName(pItem, pExpr->base.aliasName, pExpr->base.token, sizeof(pExpr->base.aliasName) - 1); - SSchema s = {0}; - s.type = (uint8_t)resType; - s.bytes = bytes; - s.colId = pExpr->base.colInfo.colId; + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); @@ -3123,7 +3120,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col insertResultField(pQueryInfo, colIndex, &ids, pUdfInfo->resBytes, pUdfInfo->resType, pExpr->base.aliasName, pExpr); } else { for (int32_t i = 0; i < ids.num; ++i) { - tscColumnListInsert(pQueryInfo->colList, index.columnIndex, uid, &s); + tscColumnListInsert(pQueryInfo->colList, index.columnIndex, uid, pSchema); } } tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); @@ -3177,6 +3174,14 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken int16_t columnIndex = COLUMN_INDEX_INITIAL_VAL; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // create tmp buf to avoid alter orginal sqlstr + strncpy(tmpTokenBuf, pToken->z, pToken->n); + pToken->z = tmpTokenBuf; + + if (pToken->type == TK_ID) { + tscRmEscapeAndTrimToken(pToken); + } + for (int16_t i = 0; i < numOfCols; ++i) { if (pToken->n != strlen(pSchema[i].name)) { continue; @@ -4227,7 +4232,11 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQuer // Append the sqlExpr into exprList of pQueryInfo structure sequentially pExpr->functionId = isValidFunction(pExpr->Expr.operand.z, pExpr->Expr.operand.n); if (pExpr->functionId < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; + SUdfInfo* pUdfInfo = NULL; + pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pExpr->Expr.operand.z, pExpr->Expr.operand.n); + if (pUdfInfo == NULL) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "invalid function name"); + } } if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false, NULL) != TSDB_CODE_SUCCESS) { @@ -5738,6 +5747,7 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq const char* msg8 = "only column in groupby clause allowed as order column"; const char* msg9 = "orderby column must projected in subquery"; const char* msg10 = "not support distinct mixed with order by"; + const char* msg11 = "not support order with udf"; setDefaultOrderInfo(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -5777,6 +5787,19 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = COLUMN_INDEX_INITIALIZER; + bool udf = false; + + if (pQueryInfo->pUdfInfo && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + int32_t usize = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + + for (int32_t i = 0; i < usize; ++i) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); + if (pUdfInfo->funcType == TSDB_UDF_TYPE_SCALAR) { + udf = true; + break; + } + } + } if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query if (getColumnIndexByName(&columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { @@ -5832,6 +5855,9 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq pQueryInfo->groupbyExpr.orderType = p1->sortOrder; pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } } else if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ @@ -5853,6 +5879,10 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } else { tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + pQueryInfo->order.order = p1->sortOrder; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; @@ -5880,9 +5910,15 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq } else if (orderByGroupbyCol){ pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = index.columnIndex; + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } } else { pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } } pItem = taosArrayGet(pSqlNode->pSortOrder, 1); @@ -5918,6 +5954,10 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq return invalidOperationMsg(pMsgBuf, msg7); } + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + tVariantListItem* p1 = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->groupbyExpr.orderIndex = pSchema[index.columnIndex].colId; pQueryInfo->groupbyExpr.orderType = p1->sortOrder; @@ -5951,6 +5991,10 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq return TSDB_CODE_SUCCESS; } + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; @@ -5963,6 +6007,10 @@ int32_t validateOrderbyNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSq return invalidOperationMsg(pMsgBuf, msg1); } + if (udf) { + return invalidOperationMsg(pMsgBuf, msg11); + } + tVariantListItem* pItem = taosArrayGet(pSqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; @@ -6273,6 +6321,13 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; SStrToken name = {.type = TK_STRING, .z = pItem->name, .n = (uint32_t)strlen(pItem->name)}; + //handle Escape character backstick + if (name.z[0] == TS_ESCAPE_CHAR && name.z[name.n - 1] == TS_ESCAPE_CHAR) { + memmove(name.z, name.z + 1, name.n); + name.z[name.n - TS_ESCAPE_CHAR_SIZE] = '\0'; + name.n -= TS_ESCAPE_CHAR_SIZE; + } + if (getColumnIndexByName(&name, pQueryInfo, &columnIndex, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(pMsg, msg17); } @@ -6606,6 +6661,9 @@ int32_t validateColumnName(char* name) { } return validateColumnName(token.z); + } else if (token.type == TK_ID) { + strRmquoteEscape(name, token.n); + return TSDB_CODE_SUCCESS; } else { if (isNumber(&token)) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -7245,7 +7303,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* const char* msg3 = "group by/session/state_window not allowed on projection query"; const char* msg4 = "retrieve tags not compatible with group by or interval query"; const char* msg5 = "functions can not be mixed up"; - const char* msg6 = "TWA/Diff/Derivative/Irate only support group by tbname"; + const char* msg6 = "TWA/Diff/Derivative/Irate/CSum/MAvg only support group by tbname"; // only retrieve tags, group by is not supportted if (tscQueryTags(pQueryInfo)) { @@ -7298,10 +7356,16 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* } if (f < 0) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * f - 1); + if (pUdfInfo->funcType == TSDB_UDF_TYPE_SCALAR) { + return invalidOperationMsg(msg, msg1); + } + continue; } - if ((!pQueryInfo->stateWindow) && (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE)) { + if ((!pQueryInfo->stateWindow) && (f == TSDB_FUNC_DIFF || f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || + f == TSDB_FUNC_IRATE || f == TSDB_FUNC_CSUM || f == TSDB_FUNC_MAVG)) { for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) { SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j); if (j == 0) { @@ -7320,6 +7384,10 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char* return invalidOperationMsg(msg, msg1); } + if (IS_SCALAR_FUNCTION(aAggs[f].status)) { + return invalidOperationMsg(msg, msg1); + } + if (f == TSDB_FUNC_COUNT && pExpr->base.colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidOperationMsg(msg, msg1); } @@ -7661,7 +7729,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j); SStrToken* pToken = &pCreateTableInfo->stableName; - + bool dbIncluded = false; char buf[TSDB_TABLE_FNAME_LEN]; SStrToken sTblToken; @@ -7721,10 +7789,19 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { for (int32_t i = 0; i < nameSize; ++i) { SStrToken* sToken = taosArrayGet(pNameList, i); + + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // create tmp buf to avoid alter orginal sqlstr + strncpy(tmpTokenBuf, sToken->z, sToken->n); + sToken->z = tmpTokenBuf; + if (TK_STRING == sToken->type) { tscDequoteAndTrimToken(sToken); } + if (TK_ID == sToken->type) { + tscRmEscapeAndTrimToken(sToken); + } + tVariantListItem* pItem = taosArrayGet(pValList, i); findColumnIndex = false; @@ -8558,12 +8635,33 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (functionId < 0) { struct SUdfInfo info = {0}; info.name = strndup(t->z, t->n); + info.keep = true; if (pQueryInfo->pUdfInfo == NULL) { pQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(struct SUdfInfo)); + } else if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + int32_t usize = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); + int32_t exist = 0; + + for (int32_t j = 0; j < usize; ++j) { + SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); + int32_t len = (int32_t)strlen(pUdfInfo->name); + if (len == t->n && strncasecmp(info.name, pUdfInfo->name, t->n) == 0) { + exist = 1; + break; + } + } + + if (exist) { + continue; + } } info.functionId = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo) * (-1) - 1;; taosArrayPush(pQueryInfo->pUdfInfo, &info); + if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 1) { + code = tscInvalidOperationMsg(pCmd->payload, "only one udf allowed", NULL); + goto _end; + } } } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 2ed8ea94b53955a2061ffe763cb0521daab30fd9..b19af46a0c7f191b84d1ea8658f13456624179c9 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1102,6 +1102,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // support only one udf if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { + if (taosArrayGetSize(pQueryInfo->pUdfInfo) > 1) { + code = tscInvalidOperationMsg(pCmd->payload, "only one udf allowed", NULL); + goto _end; + } + pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index caddde3f088c8ea65743070563a093921c3d2b2d..bb3bddeefd798366fe205eb67b55b3b4a7301df4 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -629,6 +629,10 @@ static bool hasAdditionalErrorInfo(int32_t code, SSqlCmd *pCmd) { return false; } + if (pCmd->payload == NULL) { + return false; + } + size_t len = strlen(pCmd->payload); char *z = NULL; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index f880cb11760dfdbb269ee5674a73d8e50333f905..b14c5af47a54e8917c97cec4d1e31acebda9f602 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1271,6 +1271,28 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue .pGroupList = taosArrayInit(1, POINTER_BYTES), }; + SUdfInfo* pUdfInfo = NULL; + + size_t size = tscNumOfExprs(px); + for (int32_t j = 0; j < size; ++j) { + SExprInfo* pExprInfo = tscExprGet(px, j); + + int32_t functionId = pExprInfo->base.functionId; + if (functionId < 0) { + if (pUdfInfo) { + pSql->res.code = tscInvalidOperationMsg(pSql->cmd.payload, "only one udf allowed", NULL); + return; + } + + pUdfInfo = taosArrayGet(px->pUdfInfo, -1 * functionId - 1); + int32_t code = initUdfInfo(pUdfInfo); + if (code != TSDB_CODE_SUCCESS) { + pSql->res.code = code; + return; + } + } + } + tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; @@ -1352,6 +1374,9 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute the main query while all nest queries are ready", pSql->self, pSql->self); px->pQInfo = createQInfoFromQueryNode(px, &tableGroupInfo, pSourceOperator, NULL, NULL, MASTER_SCAN, pSql->self); + px->pQInfo->runtimeEnv.udfIsCopy = true; + px->pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; + tfree(pColumnInfo); tfree(schema); @@ -4800,9 +4825,14 @@ int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaI functionId = TSDB_FUNC_STDDEV; } + SUdfInfo* pUdfInfo = NULL; + if (functionId < 0) { + pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); + } + int32_t inter = 0; getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType, - &pse->resBytes, &inter, 0, false, NULL); + &pse->resBytes, &inter, 0, false, pUdfInfo); pse->colType = pse->resType; pse->colBytes = pse->resBytes; @@ -5095,7 +5125,6 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq if (nextStr == NULL) { tstrncpy(tablename, *str, TSDB_TABLE_FNAME_LEN); - len = (int32_t) strlen(tablename); } else { len = (int32_t)(nextStr - (*str)); if (len >= TSDB_TABLE_NAME_LEN) { diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 8a82aa00a4cbc1e209b6425b4584fc0751c854f0..c91637b1e85d54fb53c55e8fab09c666263345bf 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -108,9 +108,10 @@ extern int32_t tsQuorum; extern int8_t tsUpdate; extern int8_t tsCacheLastRow; -//tsdb -extern bool tsdbForceKeepFile; -extern bool tsdbForceCompactFile; +//tsdb +extern bool tsdbForceKeepFile; +extern bool tsdbForceCompactFile; +extern int32_t tsdbWalFlushSize; // balance extern int8_t tsEnableBalance; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 0bff138fbb903cb910cd7c86b1628cf2b4f77c4e..f3ba69ec40d8ac76f8db0fc84667a1cf402bc4d0 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -155,8 +155,9 @@ int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; // tsdb config // For backward compatibility -bool tsdbForceKeepFile = false; -bool tsdbForceCompactFile = false; // compact TSDB fileset forcibly +bool tsdbForceKeepFile = false; +bool tsdbForceCompactFile = false; // compact TSDB fileset forcibly +int32_t tsdbWalFlushSize = TSDB_DEFAULT_WAL_FLUSH_SIZE; // MB // balance int8_t tsEnableBalance = 1; @@ -1652,6 +1653,17 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + // flush vnode wal file if walSize > walFlushSize and walSize > cache*0.5*blocks + cfg.option = "walFlushSize"; + cfg.ptr = &tsdbWalFlushSize; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = TSDB_MIN_WAL_FLUSH_SIZE; + cfg.maxValue = TSDB_MAX_WAL_FLUSH_SIZE; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_MB; + taosInitConfigOption(cfg); + #ifdef TD_TSZ // lossy compress cfg.option = "lossyColumns"; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java index a801f5a674acdd23f1ca7f949cbb7092f4633bda..12641087fb774a82e80c8339f752ff5f514524a0 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java @@ -9,6 +9,7 @@ public abstract class AbstractStatement extends WrapperImpl implements Statement protected List batchedArgs; private int fetchSize; + protected int affectedRows = -1; @Override public abstract ResultSet executeQuery(String sql) throws SQLException; @@ -247,6 +248,7 @@ public abstract class AbstractStatement extends WrapperImpl implements Statement public boolean getMoreResults(int current) throws SQLException { if (isClosed()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + this.affectedRows = -1; switch (current) { case Statement.CLOSE_CURRENT_RESULT: return false; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index 6565a8151e6b16d7f04b184e93dbe89d85466533..5cdaa3c70c334bc7bd97be08f2318e6fc548d22a 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -49,7 +49,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setBooleanValue(int col, boolean value) { setBoolean(col - 1, value); @@ -86,7 +86,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setByteValue(int colIndex, byte value) { setByte(colIndex - 1, value); @@ -100,7 +100,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setShortValue(int colIndex, short value) { setShort(colIndex - 1, value); @@ -114,7 +114,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setIntValue(int colIndex, int value) { setInt(colIndex - 1, value); @@ -194,7 +194,7 @@ public class TSDBResultSetRowData { /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setLongValue(int colIndex, long value) { setLong(colIndex - 1, value); @@ -262,7 +262,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setFloatValue(int colIndex, float value) { setFloat(colIndex - 1, value); @@ -302,7 +302,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setDoubleValue(int colIndex, double value) { setDouble(colIndex - 1, value); @@ -342,7 +342,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setStringValue(int colIndex, String value) { data.set(colIndex - 1, value); @@ -361,7 +361,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setByteArrayValue(int colIndex, byte[] value) { setByteArray(colIndex - 1, value); @@ -424,7 +424,7 @@ public class TSDBResultSetRowData { } /** - * $$$ this method is invoked by databaseMetaDataResultSet and so on which use a index start from 1 in JDBC api + * $$$ this method is invoked by databaseMetaDataResultSet and so on which use an index start from 1 in JDBC api */ public void setTimestampValue(int colIndex, long value) { setTimestamp(colIndex - 1, value, 0); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java index 48854e773f89a45784de3cd709ec5bbe6185e09b..0a9f017cbbd775cf710f3bac4440ee8a43403870 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java @@ -23,7 +23,7 @@ import java.util.Calendar; import java.util.Map; /* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the + * TDengine only supports a subset of the standard SQL, thus this implementation of the * standard JDBC API contains more or less some adjustments customized for certain * compatibility needs. */ diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index e1ebc4ab3cf498168181dbea08a3ac28194a5c7d..436bdcf582b821292c5f4e69f51688f9bf84b870 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -23,7 +23,6 @@ public class TSDBStatement extends AbstractStatement { * Status of current statement */ private boolean isClosed; - private int affectedRows = -1; private TSDBConnection connection; private TSDBResultSet resultSet; @@ -80,12 +79,13 @@ public class TSDBStatement extends AbstractStatement { if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); } - + // execute query long pSql = this.connection.getConnector().executeQuery(sql); // if pSql is create/insert/update/delete/alter SQL if (this.connection.getConnector().isUpdateQuery(pSql)) { - this.affectedRows = this.connection.getConnector().getAffectedRows(pSql); + int rows = this.connection.getConnector().getAffectedRows(pSql); + this.affectedRows = rows == 0 ? -1 : this.connection.getConnector().getAffectedRows(pSql); this.connection.getConnector().freeResultSet(pSql); return false; } @@ -99,7 +99,7 @@ public class TSDBStatement extends AbstractStatement { if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); } - + return this.resultSet; } @@ -113,14 +113,14 @@ public class TSDBStatement extends AbstractStatement { if (isClosed()) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); } - + if (this.connection.getConnector() == null) { throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); } - + return this.connection; } - + public void setConnection(TSDBConnection connection) { this.connection = connection; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 1ea39236b666fda106c3ee3534560b6380d7bec9..835c6f87019ddc4f3b6d7f6da7d0d101e1d087a5 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -255,8 +255,11 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { checkAvailability(columnIndex, resultSet.get(pos).size()); Object value = resultSet.get(pos).get(columnIndex - 1); - if (value == null) + if (value == null) { + wasNull = true; return null; + } + wasNull = false; if (value instanceof byte[]) return new String((byte[]) value); return value.toString(); @@ -663,4 +666,4 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { return isClosed; } -} \ No newline at end of file +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 21c76f73b287e55ef14f5d70cf6a911a9cb543db..b7f5fe8006368295753a366aa218a6cc17aa0588 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -22,7 +22,6 @@ public class RestfulStatement extends AbstractStatement { private final RestfulConnection conn; private volatile RestfulResultSet resultSet; - private volatile int affectedRows; public RestfulStatement(RestfulConnection conn, String database) { this.conn = conn; @@ -118,7 +117,7 @@ public class RestfulStatement extends AbstractStatement { throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc")); } this.resultSet = new RestfulResultSet(database, this, resultJson); - this.affectedRows = 0; + this.affectedRows = -1; return resultSet; } @@ -140,9 +139,10 @@ public class RestfulStatement extends AbstractStatement { if (head.size() != 1 || !"affected_rows".equals(head.getString(0))) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); JSONArray data = jsonObject.getJSONArray("data"); - if (data != null) - return data.getJSONArray(0).getInteger(0); - + if (data != null) { + int rows = data.getJSONArray(0).getInteger(0); + return rows == 0 ? -1 : data.getJSONArray(0).getInteger(0); + } throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } diff --git a/src/connector/python/examples/insert-lines.py b/src/connector/python/examples/insert-lines.py index 755050dfb52b180567dd80e87b63508fc4101172..1d20af7e9bcac23deb70c1dbd058bb86dd5585a5 100644 --- a/src/connector/python/examples/insert-lines.py +++ b/src/connector/python/examples/insert-lines.py @@ -1,4 +1,5 @@ import taos +from taos import SmlProtocol, SmlPrecision conn = taos.connect() dbname = "pytest_line" @@ -9,10 +10,10 @@ conn.select_db(dbname) lines = [ 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', ] -conn.schemaless_insert(lines, 0, "ns") +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) print("inserted") -conn.schemaless_insert(lines, 0, "ns") +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) result = conn.query("show tables") for row in result: diff --git a/src/connector/python/pyproject.toml b/src/connector/python/pyproject.toml index a8099199563a0e5957a7d69e75bab65cca6d17db..da61cccf49429251d49f2cba495e24e146244c85 100644 --- a/src/connector/python/pyproject.toml +++ b/src/connector/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "taos" -version = "2.1.0" +version = "2.1.1" description = "TDengine connector for python" authors = ["Taosdata Inc. "] license = "AGPL-3.0" diff --git a/src/connector/python/setup.py b/src/connector/python/setup.py index b7e10001737bc40c04173ea4a65e95248965ffda..8f1dfafe4762e4a55a6d3e7c645c945a67a10f68 100644 --- a/src/connector/python/setup.py +++ b/src/connector/python/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.1.0", + version="2.1.1", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/taos/__init__.py b/src/connector/python/taos/__init__.py index ebbad68c5a8a148a601fb5ec48f9658a1920ed62..2520984e78fad236227d9cf55c29ace92878d3bf 100644 --- a/src/connector/python/taos/__init__.py +++ b/src/connector/python/taos/__init__.py @@ -440,6 +440,7 @@ from .cursor import * from .result import * from .statement import * from .subscription import * +from .schemaless import * try: import importlib.metadata @@ -468,6 +469,8 @@ __all__ = [ "TaosRow", "TaosStmt", "PrecisionEnum", + "SmlPrecision", + "SmlProtocol" ] def connect(*args, **kwargs): diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index 1fcbf678b6a2a3f51bd757b84c08a7693166556c..4365c7eabc509f95525078378ff76d46a884c075 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -12,6 +12,7 @@ except: from .error import * from .bind import * from .field import * +from .schemaless import * # stream callback @@ -64,6 +65,8 @@ _libtaos.taos_consume.restype = ctypes.c_void_p _libtaos.taos_fetch_lengths.restype = ctypes.POINTER(ctypes.c_int) _libtaos.taos_free_result.restype = None _libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) +_libtaos.taos_schemaless_insert.restype = ctypes.c_void_p + try: _libtaos.taos_stmt_errstr.restype = c_char_p except AttributeError: @@ -808,25 +811,27 @@ def taos_stmt_use_result(stmt): return result try: - _libtaos.taos_insert_lines.restype = c_int - _libtaos.taos_insert_lines.argstype = c_void_p, c_void_p, c_int + _libtaos.taos_schemaless_insert.restype = c_void_p + _libtaos.taos_schemaless_insert.argstype = c_void_p, c_void_p, c_int, c_int, c_int except AttributeError: - print("WARNING: libtaos(%s) does not support insert_lines" % taos_get_client_info()) - - - + print("WARNING: libtaos(%s) does not support taos_schemaless_insert" % taos_get_client_info()) def taos_schemaless_insert(connection, lines, protocol, precision): - # type: (c_void_p, list[str] | tuple(str)) -> None + # type: (c_void_p, list[str] | tuple(str), SmlProtocol, SmlPrecision) -> int num_of_lines = len(lines) lines = (c_char_p(line.encode("utf-8")) for line in lines) lines_type = ctypes.c_char_p * num_of_lines p_lines = lines_type(*lines) - if precision != None: - precision = c_char_p(precision.encode("utf-8")) - errno = _libtaos.taos_schemaless_insert(connection, p_lines, num_of_lines, protocol, precision) + res = c_void_p(_libtaos.taos_schemaless_insert(connection, p_lines, num_of_lines, protocol, precision)) + errno = taos_errno(res) + affected_rows = taos_affected_rows(res) if errno != 0: - raise SchemalessError("schemaless insert error", errno) + errstr = taos_errstr(res) + taos_free_result(res) + raise SchemalessError(errstr, errno, affected_rows) + + taos_free_result(res) + return affected_rows class CTaosInterface(object): def __init__(self, config=None): diff --git a/src/connector/python/taos/connection.py b/src/connector/python/taos/connection.py index dfac42f244d19267124c5ea790d4503e28fd5a78..dc8225ab33c84930214eb8f0d8ba47f6f31a5adf 100644 --- a/src/connector/python/taos/connection.py +++ b/src/connector/python/taos/connection.py @@ -72,10 +72,9 @@ class TaosConnection(object): taos_select_db(self._conn, database) def execute(self, sql): - # type: (str) -> None + # type: (str) -> int """Simplely execute sql ignoring the results""" - res = taos_query(self._conn, sql) - taos_free_result(res) + return self.query(sql).affected_rows def query(self, sql): # type: (str) -> TaosResult @@ -118,7 +117,7 @@ class TaosConnection(object): return TaosStream(stream) def schemaless_insert(self, lines, protocol, precision): - # type: (list[str]) -> None + # type: (list[str], SmlProtocol, SmlPrecision) -> int """ 1.Line protocol and schemaless support @@ -171,6 +170,7 @@ class TaosConnection(object): conn.schemaless_insert(lines, 2, None) """ + print(lines, protocol, precision) return taos_schemaless_insert(self._conn, lines, protocol, precision) diff --git a/src/connector/python/taos/error.py b/src/connector/python/taos/error.py index 723f6f1a2db1249a3773538b4bfa6d51595a005d..122466fe3c448ec551fb910c402ad14bb6c93336 100644 --- a/src/connector/python/taos/error.py +++ b/src/connector/python/taos/error.py @@ -83,7 +83,16 @@ class ResultError(DatabaseError): class SchemalessError(DatabaseError): """taos_schemaless_insert errors.""" - pass + def __init__(self, msg=None, errno=0xffff, affected_rows=0): + DatabaseError.__init__(self, msg, errno) + self.affected_rows = affected_rows + + def __str__(self): + return self._full_msg + "(affected rows: %d)" % self.affected_rows + + # @property + # def affected_rows(self): + # return self.affected_rows class StatementError(DatabaseError): diff --git a/src/connector/python/taos/result.py b/src/connector/python/taos/result.py index 81151733615d1b7fdc3318b6e53888ae39d32b14..c9feb4d6502515cc6e3e2d4be688f2e7fcd895b2 100644 --- a/src/connector/python/taos/result.py +++ b/src/connector/python/taos/result.py @@ -123,6 +123,12 @@ class TaosResult(object): for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + + def fetch_all_into_dict(self): + """Fetch all rows and convert it to dict""" + names = [field.name for field in self.fields] + rows = self.fetch_all() + return list(dict(zip(names, row)) for row in rows) def fetch_rows_a(self, callback, param): taos_fetch_rows_a(self._result, callback, param) @@ -228,6 +234,12 @@ class TaosRow: blocks[i] = CONVERT_FUNC[fields[i].type](data, 1, field_lens[i], precision)[0] return tuple(blocks) + def as_dict(self): + values = self.as_tuple() + names = self._result.fields + dict(zip(names, values)) + + class TaosBlocks: """TDengine result blocks iterator""" diff --git a/src/connector/python/taos/schemaless.py b/src/connector/python/taos/schemaless.py new file mode 100644 index 0000000000000000000000000000000000000000..35967412f78a63e67d63f0e58bbf903f21fb275a --- /dev/null +++ b/src/connector/python/taos/schemaless.py @@ -0,0 +1,17 @@ + +class SmlPrecision: + """Schemaless timestamp precision constants""" + NOT_CONFIGURED = 0 # C.TSDB_SML_TIMESTAMP_NOT_CONFIGURED + HOURS = 1 + MINUTES = 2 + SECONDS = 3 + MILLI_SECONDS = 4 + MICRO_SECONDS = 5 + NANO_SECONDS = 6 + +class SmlProtocol: + """Schemaless protocol constants""" + UNKNOWN_PROTOCOL = 0 + LINE_PROTOCOL = 1 + TELNET_PROTOCOL = 2 + JSON_PROTOCOL = 3 \ No newline at end of file diff --git a/src/connector/python/tests/test_lines.py b/src/connector/python/tests/test_lines.py index 157580f8466ce765246184421f0756958455a54b..51d23b8e891d398b404086fdb2ff2910dcc1eb0a 100644 --- a/src/connector/python/tests/test_lines.py +++ b/src/connector/python/tests/test_lines.py @@ -1,4 +1,4 @@ -from taos.error import OperationalError +from taos.error import OperationalError, SchemalessError from taos import connect, new_bind_params, PrecisionEnum from taos import * @@ -13,35 +13,95 @@ def conn(): return connect() +def test_schemaless_insert_update_2(conn): + # type: (TaosConnection) -> None + + dbname = "test_schemaless_insert_update_2" + try: + conn.execute("drop database if exists %s" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + conn.select_db(dbname) + + lines = [ + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000', + ] + res = conn.schemaless_insert(lines, 1, 0) + print("affected rows: ", res) + assert(res == 1) + + result = conn.query("select * from st") + [before] = result.fetch_all_into_dict() + assert(before["c3"] == "passitagin, abc") + + lines = [ + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000', + ] + res = conn.schemaless_insert(lines, 1, 0) + result = conn.query("select * from st") + [after] = result.fetch_all_into_dict() + assert(after["c3"] == "passitagin") + + conn.execute("drop database if exists %s" % dbname) + conn.close() + + except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err + def test_schemaless_insert(conn): # type: (TaosConnection) -> None dbname = "pytest_taos_schemaless_insert" try: conn.execute("drop database if exists %s" % dbname) - conn.execute("create database if not exists %s precision 'us'" % dbname) + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) conn.select_db(dbname) lines = [ 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000', - 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000', + 'st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000', 'stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000', ] - conn.schemaless_insert(lines, 0, "ns") - print("inserted") + res = conn.schemaless_insert(lines, 1, 0) + print("affected rows: ", res) + assert(res == 3) lines = [ 'stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000', ] - conn.schemaless_insert(lines, 0, "ns") - print("inserted") + res = conn.schemaless_insert(lines, 1, 0) + print("affected rows: ", res) + assert(res == 1) + result = conn.query("select * from st") + + dict2 = result.fetch_all_into_dict() + print(dict2) + result.row_count + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert(result.row_count == 2) + + # error test + lines = [ + ',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000', + ] + try: + res = conn.schemaless_insert(lines, 1, 0) + print(res) + # assert(False) + except SchemalessError as err: + pass + result = conn.query("select * from st") - print(*result.fields) + result.row_count all = result.rows_iter() for row in all: print(row) result.close() - print(result.row_count) conn.execute("drop database if exists %s" % dbname) conn.close() @@ -55,3 +115,4 @@ def test_schemaless_insert(conn): if __name__ == "__main__": test_schemaless_insert(connect()) + test_schemaless_insert_update_2(connect()) diff --git a/src/connector/python/tests/test_stmt.py b/src/connector/python/tests/test_stmt.py index 938ba10eb3d2377a63f7972deb99dbd47f7de1b2..3368ecb6a9336a4295790f2cd55314ac9bb6290e 100644 --- a/src/connector/python/tests/test_stmt.py +++ b/src/connector/python/tests/test_stmt.py @@ -1,3 +1,4 @@ +# encoding:UTF-8 from taos import * from ctypes import * diff --git a/src/inc/taos.h b/src/inc/taos.h index da91ed16c8186d15e109aaf03b18c6ca4ce86837..4afec942ff991ce1009cb8c54113562f93f9c92d 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -72,6 +72,23 @@ typedef enum { SET_CONF_RET_ERR_TOO_LONG = -6 } SET_CONF_RET_CODE; +typedef enum { + TSDB_SML_UNKNOWN_PROTOCOL = 0, + TSDB_SML_LINE_PROTOCOL = 1, + TSDB_SML_TELNET_PROTOCOL = 2, + TSDB_SML_JSON_PROTOCOL = 3, +} TSDB_SML_PROTOCOL_TYPE; + +typedef enum { + TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0, + TSDB_SML_TIMESTAMP_HOURS, + TSDB_SML_TIMESTAMP_MINUTES, + TSDB_SML_TIMESTAMP_SECONDS, + TSDB_SML_TIMESTAMP_MILLI_SECONDS, + TSDB_SML_TIMESTAMP_MICRO_SECONDS, + TSDB_SML_TIMESTAMP_NANO_SECONDS, +} TSDB_SML_TIMESTAMP_TYPE; + #define RET_MSG_LENGTH 1024 typedef struct setConfRet { SET_CONF_RET_CODE retCode; @@ -188,7 +205,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); -DLL_EXPORT int taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, char* precision); +DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); #ifdef __cplusplus } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index bd1c3518a3b378368334202447ab746de8186150..78fb2ac01c1d8cd01d8d704a384891125f8bb00e 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -99,6 +99,7 @@ extern const int32_t TYPE_BYTES[15]; #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' +#define TS_ESCAPE_CHAR_SIZE 2 #define TSDB_TIME_PRECISION_MILLI 0 #define TSDB_TIME_PRECISION_MICRO 1 @@ -279,6 +280,10 @@ do { \ #define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_DEFAULT_TOTAL_BLOCKS 6 +#define TSDB_MIN_WAL_FLUSH_SIZE 128 // MB +#define TSDB_MAX_WAL_FLUSH_SIZE 10000000 // MB +#define TSDB_DEFAULT_WAL_FLUSH_SIZE 1024 // MB + #define TSDB_MIN_TABLES 4 #define TSDB_MAX_TABLES 10000000 #define TSDB_DEFAULT_TABLES 1000000 diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 4e11e4f2478fe0616701e0d183d38455b9526514..9d82245c2199b5fa0b62d709a08633e5a976b007 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -418,6 +418,9 @@ int tsdbCompact(STsdbRepo *pRepo); // no problem return true bool tsdbNoProblem(STsdbRepo* pRepo); +// unit of walSize: MB +int tsdbCheckWal(STsdbRepo *pRepo, uint32_t walSize); + #ifdef __cplusplus } #endif diff --git a/src/inc/twal.h b/src/inc/twal.h index 868a1fbd780232303b42e58185ffc00730c17546..daea34daed43a7a47ca0c50aaf759e3cb905cfef 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -66,6 +66,7 @@ int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); uint64_t walGetVersion(twalh); void walResetVersion(twalh, uint64_t newVer); +int64_t walGetFSize(twalh); #ifdef __cplusplus } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 2bc9665d7943b79e9c46a1820723a855c9c0067c..902d2adcb33f5828343037afb042c9aff7627630 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -56,6 +56,7 @@ #define REQ_EXTRA_BUF_LEN 1024 #define RESP_BUF_LEN 4096 +#define SQL_BUFF_LEN 1024 extern char configDir[]; @@ -66,6 +67,7 @@ extern char configDir[]; #define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. #define BUFFER_SIZE TSDB_MAX_ALLOWED_SQL_LEN +#define FETCH_BUFFER_SIZE 100 * TSDB_MAX_ALLOWED_SQL_LEN #define COND_BUF_LEN (BUFFER_SIZE - 30) #define COL_BUFFER_LEN ((TSDB_COL_NAME_LEN + 15) * TSDB_MAX_COLUMNS) @@ -75,6 +77,7 @@ extern char configDir[]; #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. +#define MAX_PATH_LEN 4096 #define DEFAULT_START_TIME 1500000000000 @@ -87,6 +90,7 @@ extern char configDir[]; #define FLOAT_BUFF_LEN 22 #define DOUBLE_BUFF_LEN 42 #define TIMESTAMP_BUFF_LEN 21 +#define PRINT_STAT_INTERVAL 30*1000 #define MAX_SAMPLES 10000 #define MAX_NUM_COLUMNS (TSDB_MAX_COLUMNS - 1) // exclude first column timestamp @@ -97,8 +101,10 @@ extern char configDir[]; #define MAX_QUERY_SQL_COUNT 100 #define MAX_DATABASE_COUNT 256 -#define INPUT_BUF_LEN 256 +#define MAX_JSON_BUFF 6400000 +#define INPUT_BUF_LEN 256 +#define EXTRA_SQL_LEN 256 #define TBNAME_PREFIX_LEN (TSDB_TABLE_NAME_LEN - 20) // 20 characters reserved for seq #define SMALL_BUFF_LEN 8 #define DATATYPE_BUFF_LEN (SMALL_BUFF_LEN*3) @@ -109,6 +115,45 @@ extern char configDir[]; #define DEFAULT_INTERLACE_ROWS 0 #define DEFAULT_DATATYPE_NUM 1 #define DEFAULT_CHILDTABLES 10000 +#define DEFAULT_TEST_MODE 0 +#define DEFAULT_METAFILE NULL +#define DEFAULT_SQLFILE NULL +#define DEFAULT_HOST "localhost" +#define DEFAULT_PORT 6030 +#define DEFAULT_IFACE INTERFACE_BUT +#define DEFAULT_DATABASE "test" +#define DEFAULT_REPLICA 1 +#define DEFAULT_TB_PREFIX "d" +#define DEFAULT_ESCAPE_CHAR false +#define DEFAULT_USE_METRIC true +#define DEFAULT_DROP_DB true +#define DEFAULT_AGGR_FUNC false +#define DEFAULT_DEBUG false +#define DEFAULT_VERBOSE false +#define DEFAULT_PERF_STAT false +#define DEFAULT_ANS_YES false +#define DEFAULT_OUTPUT "./output.txt" +#define DEFAULT_SYNC_MODE 0 +#define DEFAULT_DATA_TYPE {TSDB_DATA_TYPE_FLOAT,TSDB_DATA_TYPE_INT,TSDB_DATA_TYPE_FLOAT} +#define DEFAULT_DATATYPE {"FLOAT","INT","FLOAT"} +#define DEFAULT_BINWIDTH 64 +#define DEFAULT_COL_COUNT 4 +#define DEFAULT_LEN_ONE_ROW 76 +#define DEFAULT_INSERT_INTERVAL 0 +#define DEFAULT_QUERY_TIME 1 +#define DEFAULT_PREPARED_RAND 10000 +#define DEFAULT_REQ_PER_REQ 30000 +#define DEFAULT_INSERT_ROWS 10000 +#define DEFAULT_ABORT 0 +#define DEFAULT_RATIO 0 +#define DEFAULT_DISORDER_RANGE 1000 +#define DEFAULT_METHOD_DEL 1 +#define DEFAULT_TOTAL_INSERT 0 +#define DEFAULT_TOTAL_AFFECT 0 +#define DEFAULT_DEMO_MODE true +#define DEFAULT_CREATE_BATCH 10 +#define DEFAULT_SUB_INTERVAL 10000 +#define DEFAULT_QUERY_INTERVAL 10000 #define STMT_BIND_PARAM_BATCH 1 @@ -147,6 +192,7 @@ enum enum_TAOS_INTERFACE { TAOSC_IFACE, REST_IFACE, STMT_IFACE, + SML_IFACE, INTERFACE_BUT }; @@ -466,7 +512,7 @@ typedef struct SThreadInfo_S { int threadID; char db_name[TSDB_DB_NAME_LEN]; uint32_t time_precision; - char filePath[4096]; + char filePath[MAX_PATH_LEN]; FILE *fp; char tb_prefix[TSDB_TABLE_NAME_LEN]; uint64_t start_table_from; @@ -504,6 +550,7 @@ typedef struct SThreadInfo_S { uint64_t querySeq; // sequence number of sql command TAOS_SUB* tsub; + char** lines; int sockfd; } threadInfo; @@ -622,63 +669,49 @@ char *g_aggreFunc[] = {"*", "count(*)", "avg(C0)", "sum(C0)", "max(C0)", "min(C0)", "first(C0)", "last(C0)"}; SArguments g_args = { - NULL, // metaFile - 0, // test_mode - "localhost", // host - 6030, // port - INTERFACE_BUT, // iface - "root", // user -#ifdef _TD_POWER_ - "powerdb", // password -#elif (_TD_TQ_ == true) - "tqueue", // password -#elif (_TD_PRO_ == true) - "prodb", // password -#else - "taosdata", // password -#endif - "test", // database - 1, // replica - "d", // tb_prefix - false, // escapeChar - NULL, // sqlFile - true, // use_metric - true, // drop_database - false, // aggr_func - false, // debug_print - false, // verbose_print - false, // performance statistic print - false, // answer_yes; - "./output.txt", // output_file - 0, // mode : sync or async - {TSDB_DATA_TYPE_FLOAT, - TSDB_DATA_TYPE_INT, - TSDB_DATA_TYPE_FLOAT}, - { - "FLOAT", // dataType - "INT", // dataType - "FLOAT", // dataType. demo mode has 3 columns - }, - 64, // binwidth - 4, // columnCount, timestamp + float + int + float - 20 + FLOAT_BUFF_LEN + INT_BUFF_LEN + FLOAT_BUFF_LEN, // lenOfOneRow - DEFAULT_NTHREADS,// nthreads - 0, // insert_interval - DEFAULT_TIMESTAMP_STEP, // timestamp_step - 1, // query_times - 10000, // prepared_rand - DEFAULT_INTERLACE_ROWS, // interlaceRows; - 30000, // reqPerReq - (1024*1024), // max_sql_len - DEFAULT_CHILDTABLES, // ntables - 10000, // insertRows - 0, // abort - 0, // disorderRatio - 1000, // disorderRange - 1, // method_of_delete - 0, // totalInsertRows; - 0, // totalAffectedRows; - true, // demo_mode; + DEFAULT_METAFILE, // metaFile + DEFAULT_TEST_MODE, // test_mode + DEFAULT_HOST, // host + DEFAULT_PORT, // port + DEFAULT_IFACE, // iface + TSDB_DEFAULT_USER, // user + TSDB_DEFAULT_PASS, // password + DEFAULT_DATABASE, // database + DEFAULT_REPLICA, // replica + DEFAULT_TB_PREFIX, // tb_prefix + DEFAULT_ESCAPE_CHAR, // escapeChar + DEFAULT_SQLFILE, // sqlFile + DEFAULT_USE_METRIC, // use_metric + DEFAULT_DROP_DB, // drop_database + DEFAULT_AGGR_FUNC, // aggr_func + DEFAULT_DEBUG, // debug_print + DEFAULT_VERBOSE, // verbose_print + DEFAULT_PERF_STAT, // performance statistic print + DEFAULT_ANS_YES, // answer_yes; + DEFAULT_OUTPUT, // output_file + DEFAULT_SYNC_MODE, // mode : sync or async + DEFAULT_DATA_TYPE, // data_type + DEFAULT_DATATYPE, // dataType + DEFAULT_BINWIDTH, // binwidth + DEFAULT_COL_COUNT, // columnCount, timestamp + float + int + float + DEFAULT_LEN_ONE_ROW, // lenOfOneRow + DEFAULT_NTHREADS, // nthreads + DEFAULT_INSERT_INTERVAL, // insert_interval + DEFAULT_TIMESTAMP_STEP, // timestamp_step + DEFAULT_QUERY_TIME, // query_times + DEFAULT_PREPARED_RAND, // prepared_rand + DEFAULT_INTERLACE_ROWS, // interlaceRows; + DEFAULT_REQ_PER_REQ, // reqPerReq + TSDB_MAX_ALLOWED_SQL_LEN, // max_sql_len + DEFAULT_CHILDTABLES, // ntables + DEFAULT_INSERT_ROWS, // insertRows + DEFAULT_ABORT, // abort + DEFAULT_RATIO, // disorderRatio + DEFAULT_DISORDER_RANGE, // disorderRange + DEFAULT_METHOD_DEL, // method_of_delete + DEFAULT_TOTAL_INSERT, // totalInsertRows; + DEFAULT_TOTAL_AFFECT, // totalAffectedRows; + DEFAULT_DEMO_MODE, // demo_mode; }; static SDbs g_Dbs; @@ -733,7 +766,7 @@ static FILE * g_fpOfInsertResult = NULL; /////////////////////////////////////////////////// -static void ERROR_EXIT(const char *msg) { errorPrint("%s", msg); exit(-1); } +static void ERROR_EXIT(const char *msg) { errorPrint("%s", msg); exit(EXIT_FAILURE); } #ifndef TAOSDEMO_COMMIT_SHA1 #define TAOSDEMO_COMMIT_SHA1 "unknown" @@ -1055,6 +1088,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->iface = REST_IFACE; } else if (0 == strcasecmp(argv[i+1], "stmt")) { arguments->iface = STMT_IFACE; + } else if (0 == strcasecmp(argv[i+1], "sml")) { + arguments->iface = SML_IFACE; } else { errorWrongValue(argv[0], "-I", argv[i+1]); exit(EXIT_FAILURE); @@ -1067,6 +1102,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->iface = REST_IFACE; } else if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "stmt")) { arguments->iface = STMT_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "sml")) { + arguments->iface = SML_IFACE; } else { errorPrintReqArg3(argv[0], "--interface"); exit(EXIT_FAILURE); @@ -1078,6 +1115,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->iface = REST_IFACE; } else if (0 == strcasecmp((char *)(argv[i] + strlen("-I")), "stmt")) { arguments->iface = STMT_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("-I")), "sml")) { + arguments->iface = SML_IFACE; } else { errorWrongValue(argv[0], "-I", (char *)(argv[i] + strlen("-I"))); @@ -1094,6 +1133,8 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->iface = REST_IFACE; } else if (0 == strcasecmp(argv[i+1], "stmt")) { arguments->iface = STMT_IFACE; + } else if (0 == strcasecmp(argv[i+1], "sml")) { + arguments->iface = SML_IFACE; } else { errorWrongValue(argv[0], "--interface", argv[i+1]); exit(EXIT_FAILURE); @@ -2000,7 +2041,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } g_args.columnCount = columnCount; - g_args.lenOfOneRow = 20; // timestamp + g_args.lenOfOneRow = TIMESTAMP_BUFF_LEN; // timestamp for (int c = 0; c < g_args.columnCount; c++) { switch(g_args.data_type[c]) { case TSDB_DATA_TYPE_BINARY: @@ -2162,7 +2203,7 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { int num_fields = taos_field_count(res); TAOS_FIELD *fields = taos_fetch_fields(res); - char* databuf = (char*) calloc(1, 100*1024*1024); + char* databuf = (char*) calloc(1, FETCH_BUFFER_SIZE); if (databuf == NULL) { errorPrint2("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", __func__, __LINE__); @@ -2173,11 +2214,11 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { // fetch the records row by row while((row = taos_fetch_row(res))) { - if (totalLen >= (100*1024*1024 - HEAD_BUFF_LEN*2)) { + if (totalLen >= (FETCH_BUFFER_SIZE - HEAD_BUFF_LEN*2)) { if (strlen(pThreadInfo->filePath) > 0) appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; - memset(databuf, 0, 100*1024*1024); + memset(databuf, 0, FETCH_BUFFER_SIZE); } num_rows++; char temp[HEAD_BUFF_LEN] = {0}; @@ -2239,7 +2280,7 @@ static int32_t rand_bool() { static int cursor; cursor++; if (cursor > (g_args.prepared_rand - 1)) cursor = 0; - return g_randint[cursor % g_args.prepared_rand] % 2; + return g_randint[cursor % g_args.prepared_rand] % TSDB_DATA_BOOL_NULL; } static char *rand_tinyint_str() @@ -2256,7 +2297,7 @@ static int32_t rand_tinyint() static int cursor; cursor++; if (cursor > (g_args.prepared_rand - 1)) cursor = 0; - return g_randint[cursor % g_args.prepared_rand] % 128; + return g_randint[cursor % g_args.prepared_rand] % TSDB_DATA_TINYINT_NULL; } static char *rand_utinyint_str() @@ -2273,7 +2314,7 @@ static int32_t rand_utinyint() static int cursor; cursor++; if (cursor > (g_args.prepared_rand - 1)) cursor = 0; - return g_randuint[cursor % g_args.prepared_rand] % 255; + return g_randuint[cursor % g_args.prepared_rand] % TSDB_DATA_UTINYINT_NULL; } static char *rand_smallint_str() @@ -2290,7 +2331,7 @@ static int32_t rand_smallint() static int cursor; cursor++; if (cursor > (g_args.prepared_rand - 1)) cursor = 0; - return g_randint[cursor % g_args.prepared_rand] % 32768; + return g_randint[cursor % g_args.prepared_rand] % TSDB_DATA_SMALLINT_NULL; } static char *rand_usmallint_str() @@ -2307,7 +2348,7 @@ static int32_t rand_usmallint() static int cursor; cursor++; if (cursor > (g_args.prepared_rand - 1)) cursor = 0; - return g_randuint[cursor % g_args.prepared_rand] % 65535; + return g_randuint[cursor % g_args.prepared_rand] % TSDB_DATA_USMALLINT_NULL; } static char *rand_int_str() @@ -2468,7 +2509,7 @@ static void rand_string(char *str, int size) { //--size; int n; for (n = 0; n < size; n++) { - int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1); + int key = abs(taosRandom()) % (int)(sizeof(charset) - 1); str[n] = charset[key]; } str[n] = 0; @@ -2611,7 +2652,8 @@ static int printfInsertMeta() { // first time if no iface specified printf("interface: \033[33m%s\033[0m\n", (g_args.iface==TAOSC_IFACE)?"taosc": - (g_args.iface==REST_IFACE)?"rest":"stmt"); + (g_args.iface==REST_IFACE)?"rest": + (g_args.iface==STMT_IFACE)?"stmt":"sml"); } printf("host: \033[33m%s:%u\033[0m\n", @@ -2737,7 +2779,8 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].dataSource); printf(" iface: \033[33m%s\033[0m\n", (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest": + (g_Dbs.db[i].superTbls[j].iface==STMT_IFACE)?"stmt":"sml"); if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].childTblLimit); @@ -2936,7 +2979,8 @@ static void printfInsertMetaToFile(FILE* fp) { g_Dbs.db[i].superTbls[j].dataSource); fprintf(fp, " iface: %s\n", (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest": + (g_Dbs.db[i].superTbls[j].iface==STMT_IFACE)?"stmt":"sml"); fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); fprintf(fp, " interlace rows: %u\n", @@ -3353,7 +3397,7 @@ static void printfDbInfoForQueryToFile( static void printfQuerySystemInfo(TAOS * taos) { char filename[MAX_FILE_NAME_LEN] = {0}; - char buffer[1024] = {0}; + char buffer[SQL_BUFF_LEN] = {0}; TAOS_RES* res; time_t t; @@ -3392,12 +3436,12 @@ static void printfQuerySystemInfo(TAOS * taos) { printfDbInfoForQueryToFile(filename, dbInfos[i], i); // show db.vgroups - snprintf(buffer, 1024, "show %s.vgroups;", dbInfos[i]->name); + snprintf(buffer, SQL_BUFF_LEN, "show %s.vgroups;", dbInfos[i]->name); res = taos_query(taos, buffer); xDumpResultToFile(filename, res); // show db.stables - snprintf(buffer, 1024, "show %s.stables;", dbInfos[i]->name); + snprintf(buffer, SQL_BUFF_LEN, "show %s.stables;", dbInfos[i]->name); res = taos_query(taos, buffer); xDumpResultToFile(filename, res); free(dbInfos[i]); @@ -3438,8 +3482,14 @@ static int postProceSql(char *host, uint16_t port, 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; - snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", - g_Dbs.user, g_Dbs.password); + if (g_args.test_mode == INSERT_TEST) { + snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", + g_Dbs.user, g_Dbs.password); + } else { + snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", + g_queryInfo.user, g_queryInfo.password); + } + size_t userpass_buf_len = strlen(userpass_buf); size_t encoded_len = 4 * ((userpass_buf_len +2) / 3); @@ -3504,7 +3554,7 @@ static int postProceSql(char *host, uint16_t port, do { #ifdef WINDOWS - bytes = recv(pThreadInfo->sockfds, response_buf + received, resp_len - received, 0); + bytes = recv(pThreadInfo->sockfd, response_buf + received, resp_len - received, 0); #else bytes = read(pThreadInfo->sockfd, response_buf + received, resp_len - received); #endif @@ -3517,18 +3567,22 @@ static int postProceSql(char *host, uint16_t port, break; received += bytes; - verbosePrint("%s() LN%d: received:%d resp_len:%d, response_buf:\n%s\n", - __func__, __LINE__, received, resp_len, response_buf); + response_buf[RESP_BUF_LEN - 1] = '\0'; - if (((NULL != strstr(response_buf, resEncodingChunk)) - && (NULL != strstr(response_buf, resHttp))) - || ((NULL != strstr(response_buf, resHttpOk)) - && (NULL != strstr(response_buf, "\"status\":")))) { - debugPrint( - "%s() LN%d: received:%d resp_len:%d, response_buf:\n%s\n", + if (strlen(response_buf)) { + verbosePrint("%s() LN%d: received:%d resp_len:%d, response_buf:\n%s\n", __func__, __LINE__, received, resp_len, response_buf); - break; - } + + if (((NULL != strstr(response_buf, resEncodingChunk)) + && (NULL != strstr(response_buf, resHttp))) + || ((NULL != strstr(response_buf, resHttpOk)) + && (NULL != strstr(response_buf, "\"status\":")))) { + debugPrint( + "%s() LN%d: received:%d resp_len:%d, response_buf:\n%s\n", + __func__, __LINE__, received, resp_len, response_buf); + break; + } + } } while(received < resp_len); if (received == resp_len) { @@ -3536,8 +3590,6 @@ static int postProceSql(char *host, uint16_t port, ERROR_EXIT("storing complete response from socket"); } - response_buf[RESP_BUF_LEN - 1] = '\0'; - if (strlen(pThreadInfo->filePath) > 0) { appendResultBufToFile(response_buf, pThreadInfo); } @@ -3740,7 +3792,7 @@ static int calcRowLen(SSuperTable* superTbls) { } } - superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + superTbls->lenOfOneRow = lenOfOneRow + TIMESTAMP_BUFF_LEN; // timestamp int tagIndex; int lenOfTagOfOneRow = 0; @@ -3794,7 +3846,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, char* dbName, char* stbName, char** childTblNameOfSuperTbl, int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset, bool escapChar) { - char command[1024] = "\0"; + char command[SQL_BUFF_LEN] = "\0"; char limitBuf[100] = "\0"; TAOS_RES * res; @@ -3806,7 +3858,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, limit, offset); //get all child table name use cmd: select tbname from superTblName; - snprintf(command, 1024, escapChar ? "select tbname from %s.`%s` %s" : + snprintf(command, SQL_BUFF_LEN, escapChar ? "select tbname from %s.`%s` %s" : "select tbname from %s.%s %s", dbName, stbName, limitBuf); res = taos_query(taos, command); @@ -3814,12 +3866,12 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, if (code != 0) { taos_free_result(res); taos_close(taos); - errorPrint2("%s() LN%d, failed to run command %s\n", - __func__, __LINE__, command); + errorPrint2("%s() LN%d, failed to run command %s, reason: %s\n", + __func__, __LINE__, command, taos_errstr(res)); exit(EXIT_FAILURE); } - int64_t childTblCount = (limit < 0)?10000:limit; + int64_t childTblCount = (limit < 0)?DEFAULT_CHILDTABLES:limit; int64_t count = 0; if (childTblName == NULL) { childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); @@ -3884,17 +3936,17 @@ static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { - char command[1024] = "\0"; + char command[SQL_BUFF_LEN] = "\0"; TAOS_RES * res; TAOS_ROW row = NULL; int count = 0; //get schema use cmd: describe superTblName; - snprintf(command, 1024, "describe %s.%s", dbName, superTbls->stbName); + snprintf(command, SQL_BUFF_LEN, "describe %s.%s", dbName, superTbls->stbName); res = taos_query(taos, command); int32_t code = taos_errno(res); if (code != 0) { - printf("failed to run command %s\n", command); + printf("failed to run command %s, reason: %s\n", command, taos_errstr(res)); taos_free_result(res); return -1; } @@ -4220,10 +4272,10 @@ static int createSuperTable( } } - superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp + superTbl->lenOfOneRow = lenOfOneRow + TIMESTAMP_BUFF_LEN; // timestamp // save for creating child table - superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1); + superTbl->colsOfCreateChildTable = (char*)calloc(len+TIMESTAMP_BUFF_LEN, 1); if (NULL == superTbl->colsOfCreateChildTable) { taos_close(taos); free(command); @@ -4232,7 +4284,7 @@ static int createSuperTable( exit(EXIT_FAILURE); } - snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols); + snprintf(superTbl->colsOfCreateChildTable, len+TIMESTAMP_BUFF_LEN, "(ts timestamp%s)", cols); verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbl->colsOfCreateChildTable); @@ -4335,7 +4387,7 @@ static int createSuperTable( superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow; - + snprintf(command, BUFFER_SIZE, superTbl->escapeChar ? "CREATE TABLE IF NOT EXISTS %s.`%s` (ts TIMESTAMP%s) TAGS %s": @@ -4467,6 +4519,10 @@ int createDatabasesAndStables(char *command) { int validStbCount = 0; for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (g_Dbs.db[i].superTbls[j].iface == SML_IFACE) { + goto skip; + } + sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].stbName); ret = queryDbExec(taos, command, NO_INSERT_TYPE, true); @@ -4488,6 +4544,7 @@ int createDatabasesAndStables(char *command) { continue; } } + skip: validStbCount ++; } g_Dbs.db[i].superTblCount = validStbCount; @@ -4525,7 +4582,7 @@ static void* createTable(void *sarg) i <= pThreadInfo->end_table_to; i++) { if (0 == g_Dbs.use_metric) { snprintf(pThreadInfo->buffer, buff_len, - g_args.escapeChar ? + g_args.escapeChar ? "CREATE TABLE IF NOT EXISTS %s.`%s%"PRIu64"` %s;" : "CREATE TABLE IF NOT EXISTS %s.%s%"PRIu64" %s;", pThreadInfo->db_name, @@ -4574,7 +4631,7 @@ static void* createTable(void *sarg) batchNum++; if ((batchNum < stbInfo->batchCreateTableNum) && ((buff_len - len) - >= (stbInfo->lenOfTagOfOneRow + 256))) { + >= (stbInfo->lenOfTagOfOneRow + EXTRA_SQL_LEN))) { continue; } } @@ -4590,7 +4647,7 @@ static void* createTable(void *sarg) } pThreadInfo->tables_created += batchNum; uint64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { + if (currentPrintTime - lastPrintTime > PRINT_STAT_INTERVAL) { printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n", pThreadInfo->threadID, pThreadInfo->start_table_from, i); lastPrintTime = currentPrintTime; @@ -4762,7 +4819,7 @@ static int readTagFromCsvFileToMem(SSuperTable * stbInfo) { stbInfo->tagDataBuf = NULL; } - int tagCount = 10000; + int tagCount = MAX_SAMPLES; int count = 0; char* tagDataBuf = calloc(1, stbInfo->lenOfTagOfOneRow * tagCount); if (tagDataBuf == NULL) { @@ -5168,35 +5225,35 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (port && port->type == cJSON_Number) { g_Dbs.port = port->valueint; } else if (!port) { - g_Dbs.port = 6030; + g_Dbs.port = DEFAULT_PORT; } cJSON* user = cJSON_GetObjectItem(root, "user"); if (user && user->type == cJSON_String && user->valuestring != NULL) { tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE); } else if (!user) { - tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE); + tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); } cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { tstrncpy(g_Dbs.password, password->valuestring, SHELL_MAX_PASSWORD_LEN); } else if (!password) { - tstrncpy(g_Dbs.password, "taosdata", SHELL_MAX_PASSWORD_LEN); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN); } cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); } else if (!resultfile) { - tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); + tstrncpy(g_Dbs.resultFile, DEFAULT_OUTPUT, MAX_FILE_NAME_LEN); } cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); if (threads && threads->type == cJSON_Number) { g_Dbs.threadCount = threads->valueint; } else if (!threads) { - g_Dbs.threadCount = 1; + g_Dbs.threadCount = DEFAULT_NTHREADS; } else { errorPrint("%s", "failed to read json, threads not found\n"); goto PARSE_OVER; @@ -5206,7 +5263,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (threads2 && threads2->type == cJSON_Number) { g_Dbs.threadCountForCreateTbl = threads2->valueint; } else if (!threads2) { - g_Dbs.threadCountForCreateTbl = 1; + g_Dbs.threadCountForCreateTbl = DEFAULT_NTHREADS; } else { errorPrint("%s", "failed to read json, threads2 not found\n"); goto PARSE_OVER; @@ -5220,7 +5277,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_args.insert_interval = gInsertInterval->valueint; } else if (!gInsertInterval) { - g_args.insert_interval = 0; + g_args.insert_interval = DEFAULT_INSERT_INTERVAL; } else { errorPrint("%s", "failed to read json, insert_interval input mistake\n"); goto PARSE_OVER; @@ -5235,7 +5292,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_args.interlaceRows = interlaceRows->valueint; } else if (!interlaceRows) { - g_args.interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + g_args.interlaceRows = DEFAULT_INTERLACE_ROWS; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { errorPrint("%s", "failed to read json, interlaceRows input mistake\n"); goto PARSE_OVER; @@ -5250,7 +5307,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_args.max_sql_len = maxSqlLen->valueint; } else if (!maxSqlLen) { - g_args.max_sql_len = (1024*1024); + g_args.max_sql_len = TSDB_MAX_ALLOWED_SQL_LEN; } else { errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__); @@ -5289,7 +5346,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_args.prepared_rand = prepareRand->valueint; } else if (!prepareRand) { - g_args.prepared_rand = 10000; + g_args.prepared_rand = DEFAULT_PREPARED_RAND; } else { errorPrint("%s() LN%d, failed to read json, prepared_rand not found\n", __func__, __LINE__); @@ -5305,7 +5362,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { g_args.answer_yes = true; } else { - g_args.answer_yes = false; + g_args.answer_yes = DEFAULT_ANS_YES; } } else if (!answerPrompt) { g_args.answer_yes = true; // default is no, mean answer_yes. @@ -5604,7 +5661,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; } else if (!batchCreateTbl) { - g_Dbs.db[i].superTbls[j].batchCreateTableNum = 10; + g_Dbs.db[i].superTbls[j].batchCreateTableNum = DEFAULT_CREATE_BATCH; } else { errorPrint("%s", "failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; @@ -5667,6 +5724,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { g_Dbs.db[i].superTbls[j].iface= REST_IFACE; } else if (0 == strcasecmp(stbIface->valuestring, "stmt")) { g_Dbs.db[i].superTbls[j].iface= STMT_IFACE; + } else if (0 == strcasecmp(stbIface->valuestring, "sml")) { + g_Dbs.db[i].superTbls[j].iface= SML_IFACE; + g_args.iface = SML_IFACE; } else { errorPrint("failed to read json, insert_mode %s not recognized\n", stbIface->valuestring); @@ -5883,7 +5943,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (disorderRange && disorderRange->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; } else if (!disorderRange) { - g_Dbs.db[i].superTbls[j].disorderRange = 1000; + g_Dbs.db[i].superTbls[j].disorderRange = DEFAULT_DISORDER_RANGE; } else { errorPrint("%s", "failed to read json, disorderRange not found\n"); goto PARSE_OVER; @@ -5931,7 +5991,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (host && host->type == cJSON_String && host->valuestring != NULL) { tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE); } else if (!host) { - tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + tstrncpy(g_queryInfo.host, DEFAULT_HOST, MAX_HOSTNAME_SIZE); } else { errorPrint("%s", "failed to read json, host not found\n"); goto PARSE_OVER; @@ -5941,21 +6001,21 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (port && port->type == cJSON_Number) { g_queryInfo.port = port->valueint; } else if (!port) { - g_queryInfo.port = 6030; + g_queryInfo.port = DEFAULT_PORT; } cJSON* user = cJSON_GetObjectItem(root, "user"); if (user && user->type == cJSON_String && user->valuestring != NULL) { tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); } else if (!user) { - tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ; + tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); ; } cJSON* password = cJSON_GetObjectItem(root, "password"); if (password && password->type == cJSON_String && password->valuestring != NULL) { tstrncpy(g_queryInfo.password, password->valuestring, SHELL_MAX_PASSWORD_LEN); } else if (!password) { - tstrncpy(g_queryInfo.password, "taosdata", SHELL_MAX_PASSWORD_LEN);; + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, SHELL_MAX_PASSWORD_LEN);; } cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, @@ -5983,7 +6043,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } g_args.query_times = gQueryTimes->valueint; } else if (!gQueryTimes) { - g_args.query_times = 1; + g_args.query_times = DEFAULT_QUERY_TIME; } else { errorPrint("%s", "failed to read json, query_times input mistake\n"); goto PARSE_OVER; @@ -6081,7 +6141,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!interval) { //printf("failed to read json, subscribe interval no found\n"); //goto PARSE_OVER; - g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000; + g_queryInfo.specifiedQueryInfo.subscribeInterval = DEFAULT_SUB_INTERVAL; } cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); @@ -6228,7 +6288,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } g_queryInfo.superQueryInfo.threadCnt = threads->valueint; } else if (!threads) { - g_queryInfo.superQueryInfo.threadCnt = 1; + g_queryInfo.superQueryInfo.threadCnt = DEFAULT_NTHREADS; } //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count"); @@ -6273,7 +6333,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!superInterval) { //printf("failed to read json, subscribe interval no found\n"); //goto PARSE_OVER; - g_queryInfo.superQueryInfo.subscribeInterval = 10000; + g_queryInfo.superQueryInfo.subscribeInterval = DEFAULT_QUERY_INTERVAL; } cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart"); @@ -6393,7 +6453,7 @@ static bool getInfoFromJsonFile(char* file) { } bool ret = false; - int maxLen = 6400000; + int maxLen = MAX_JSON_BUFF; char *content = calloc(1, maxLen + 1); int len = fread(content, 1, maxLen, fp); if (len <= 0) { @@ -6551,7 +6611,7 @@ static int getRowDataFromSample( stbInfo->sampleDataBuf + stbInfo->lenOfOneRow * (*sampleUsePos)); } - + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); (*sampleUsePos)++; @@ -7007,7 +7067,8 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) { int32_t affectedRows; SSuperTable* stbInfo = pThreadInfo->stbInfo; - + TAOS_RES* res; + int32_t code; uint16_t iface; if (stbInfo) iface = stbInfo->iface; @@ -7059,7 +7120,16 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) } affectedRows = k; break; - + case SML_IFACE: + res = taos_schemaless_insert(pThreadInfo->taos, pThreadInfo->lines, k, 0, pThreadInfo->time_precision); + code = taos_errno(res); + affectedRows = taos_affected_rows(res); + if (code != TSDB_CODE_SUCCESS) { + errorPrint2("%s() LN%d, failed to execute schemaless insert. reason: %s\n", + __func__, __LINE__, taos_errstr(res)); + exit(EXIT_FAILURE); + } + break; default: errorPrint2("%s() LN%d: unknown insert mode: %d\n", __func__, __LINE__, stbInfo->iface); @@ -7076,7 +7146,7 @@ static void getTableName(char *pTblName, if (stbInfo) { if (AUTO_CREATE_SUBTBL != stbInfo->autoCreateTable) { if (stbInfo->childTblLimit > 0) { - snprintf(pTblName, TSDB_TABLE_NAME_LEN, + snprintf(pTblName, TSDB_TABLE_NAME_LEN, stbInfo->escapeChar ? "`%s`" : "%s", stbInfo->childTblName + (tableSeq - stbInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); @@ -7089,12 +7159,12 @@ static void getTableName(char *pTblName, stbInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN); } } else { - snprintf(pTblName, TSDB_TABLE_NAME_LEN, + snprintf(pTblName, TSDB_TABLE_NAME_LEN, stbInfo->escapeChar ? "`%s%"PRIu64"`" : "%s%"PRIu64"", stbInfo->childTblPrefix, tableSeq); } } else { - snprintf(pTblName, TSDB_TABLE_NAME_LEN, + snprintf(pTblName, TSDB_TABLE_NAME_LEN, g_args.escapeChar ? "`%s%"PRIu64"`" : "%s%"PRIu64"", g_args.tb_prefix, tableSeq); } @@ -9545,6 +9615,444 @@ free_of_interlace_stmt: #endif +static void generateSmlHead(char* smlHead, SSuperTable* stbInfo, threadInfo* pThreadInfo, int tbSeq) { + int64_t dataLen = 0; + dataLen += snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "%s,id=%s%" PRIu64 "", stbInfo->stbName, + stbInfo->childTblPrefix, + tbSeq + pThreadInfo->start_table_from); + for (int j = 0; j < stbInfo->tagCount; j++) { + tstrncpy(smlHead + dataLen, ",", 2); + dataLen += 1; + switch (stbInfo->tags[j].data_type) { + case TSDB_DATA_TYPE_TIMESTAMP: + errorPrint2( + "%s() LN%d, Does not support data type %s as tag\n", + __func__, __LINE__, stbInfo->tags[j].dataType); + exit(EXIT_FAILURE); + case TSDB_DATA_TYPE_BOOL: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%s", j, rand_bool_str()); + break; + case TSDB_DATA_TYPE_TINYINT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%si8", j, rand_tinyint_str()); + break; + case TSDB_DATA_TYPE_UTINYINT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%su8", j, rand_utinyint_str()); + break; + case TSDB_DATA_TYPE_SMALLINT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%si16", j, rand_smallint_str()); + break; + case TSDB_DATA_TYPE_USMALLINT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%su16", j, rand_usmallint_str()); + break; + case TSDB_DATA_TYPE_INT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%si32", j, rand_int_str()); + break; + case TSDB_DATA_TYPE_UINT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%su32", j, rand_uint_str()); + break; + case TSDB_DATA_TYPE_BIGINT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%si64", j, rand_bigint_str()); + break; + case TSDB_DATA_TYPE_UBIGINT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%su64", j, rand_ubigint_str()); + break; + case TSDB_DATA_TYPE_FLOAT: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%sf32", j, rand_float_str()); + break; + case TSDB_DATA_TYPE_DOUBLE: + dataLen += + snprintf(smlHead + dataLen, HEAD_BUFF_LEN - dataLen, + "T%d=%sf64", j, rand_double_str()); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (stbInfo->tags[j].dataLen > TSDB_MAX_BINARY_LEN) { + errorPrint2( + "binary or nchar length overflow, maxsize:%u\n", + (uint32_t)TSDB_MAX_BINARY_LEN); + exit(EXIT_FAILURE); + } + char *buf = (char *)calloc(stbInfo->tags[j].dataLen + 1, 1); + if (NULL == buf) { + errorPrint2("calloc failed! size:%d\n", + stbInfo->tags[j].dataLen); + exit(EXIT_FAILURE); + } + rand_string(buf, stbInfo->tags[j].dataLen); + if (stbInfo->tags[j].data_type == TSDB_DATA_TYPE_BINARY) { + dataLen += snprintf(smlHead + dataLen, + HEAD_BUFF_LEN - dataLen, + "T%d=\"%s\"", j, buf); + } else { + dataLen += snprintf(smlHead + dataLen, + HEAD_BUFF_LEN - dataLen, + "T%d=L\"%s\"", j, buf); + } + tmfree(buf); + break; + + default: + errorPrint2("%s() LN%d, Unknown data type %s\n", __func__, + __LINE__, stbInfo->tags[j].dataType); + exit(EXIT_FAILURE); + } + } +} + +static void generateSmlTail(char* line, char* smlHead, SSuperTable* stbInfo, + threadInfo* pThreadInfo, int64_t timestamp) { + int dataLen = 0; + dataLen = snprintf(line, BUFFER_SIZE, "%s ", smlHead); + for (uint32_t c = 0; c < stbInfo->columnCount; c++) { + if (c != 0) { + tstrncpy(line + dataLen, ",", 2); + dataLen += 1; + } + switch (stbInfo->columns[c].data_type) { + case TSDB_DATA_TYPE_TIMESTAMP: + errorPrint2( + "%s() LN%d, Does not support data type %s as tag\n", + __func__, __LINE__, stbInfo->columns[c].dataType); + exit(EXIT_FAILURE); + case TSDB_DATA_TYPE_BOOL: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, "c%d=%s", + c, rand_bool_str()); + break; + case TSDB_DATA_TYPE_TINYINT: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, "c%d=%si8", + c, rand_tinyint_str()); + break; + case TSDB_DATA_TYPE_UTINYINT: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, "c%d=%su8", + c, rand_utinyint_str()); + break; + case TSDB_DATA_TYPE_SMALLINT: + dataLen += snprintf( + line + dataLen, BUFFER_SIZE - dataLen, + "c%d=%si16", c, rand_smallint_str()); + break; + case TSDB_DATA_TYPE_USMALLINT: + dataLen += snprintf( + line + dataLen, BUFFER_SIZE - dataLen, + "c%d=%su16", c, rand_usmallint_str()); + break; + case TSDB_DATA_TYPE_INT: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=%si32", c, rand_int_str()); + break; + case TSDB_DATA_TYPE_UINT: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=%su32", c, rand_uint_str()); + break; + case TSDB_DATA_TYPE_BIGINT: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=%si64", c, rand_bigint_str()); + break; + case TSDB_DATA_TYPE_UBIGINT: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=%su64", c, rand_ubigint_str()); + break; + case TSDB_DATA_TYPE_FLOAT: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=%sf32", c, rand_float_str()); + break; + case TSDB_DATA_TYPE_DOUBLE: + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=%sf64", c, rand_double_str()); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (stbInfo->columns[c].dataLen > TSDB_MAX_BINARY_LEN) { + errorPrint2( + "binary or nchar length overflow, maxsize:%u\n", + (uint32_t)TSDB_MAX_BINARY_LEN); + exit(EXIT_FAILURE); + } + char *buf = + (char *)calloc(stbInfo->columns[c].dataLen + 1, 1); + if (NULL == buf) { + errorPrint2("calloc failed! size:%d\n", + stbInfo->columns[c].dataLen); + exit(EXIT_FAILURE); + } + rand_string(buf, stbInfo->columns[c].dataLen); + if (stbInfo->columns[c].data_type == + TSDB_DATA_TYPE_BINARY) { + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=\"%s\"", c, buf); + } else { + dataLen += snprintf(line + dataLen, + BUFFER_SIZE - dataLen, + "c%d=L\"%s\"", c, buf); + } + tmfree(buf); + break; + default: + errorPrint2("%s() LN%d, Unknown data type %s\n", + __func__, __LINE__, + stbInfo->columns[c].dataType); + exit(EXIT_FAILURE); + } + } + dataLen += snprintf(line + dataLen, BUFFER_SIZE - dataLen," %" PRId64 "", timestamp); +} + +static void* syncWriteInterlaceSml(threadInfo *pThreadInfo, uint32_t interlaceRows) { + debugPrint("[%d] %s() LN%d: ### interlace schemaless write\n", + pThreadInfo->threadID, __func__, __LINE__); + int64_t insertRows; + uint64_t maxSqlLen; + int64_t timeStampStep; + uint64_t insert_interval; + + SSuperTable* stbInfo = pThreadInfo->stbInfo; + + if (stbInfo) { + insertRows = stbInfo->insertRows; + maxSqlLen = stbInfo->maxSqlLen; + timeStampStep = stbInfo->timeStampStep; + insert_interval = stbInfo->insertInterval; + } else { + insertRows = g_args.insertRows; + maxSqlLen = g_args.max_sql_len; + timeStampStep = g_args.timestamp_step; + insert_interval = g_args.insert_interval; + } + + debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, insertRows); + + if (interlaceRows > g_args.reqPerReq) + interlaceRows = g_args.reqPerReq; + + uint32_t batchPerTbl = interlaceRows; + uint32_t batchPerTblTimes; + + if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { + batchPerTblTimes = + g_args.reqPerReq / interlaceRows; + } else { + batchPerTblTimes = 1; + } + + char **smlHeadList = calloc(pThreadInfo->ntables, sizeof(char *)); + assert(smlHeadList); + for (int t = 0; t < pThreadInfo->ntables; t++) { + char* smlHead = *((char **)smlHeadList + t * sizeof(char *)); + smlHead = (char *)calloc(HEAD_BUFF_LEN, 1); + if (NULL == smlHead) { + errorPrint2("calloc failed! size:%d\n", HEAD_BUFF_LEN); + exit(EXIT_FAILURE); + } + generateSmlHead(smlHead, stbInfo, pThreadInfo, t); + + } + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + uint64_t st = 0; + uint64_t et = UINT64_MAX; + + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + uint64_t endTs; + + uint64_t tableSeq = pThreadInfo->start_table_from; + int64_t startTime = pThreadInfo->start_time; + + uint64_t generatedRecPerTbl = 0; + bool flagSleep = true; + uint64_t sleepTimeTotal = 0; + + int percentComplete = 0; + int64_t totalRows = insertRows * pThreadInfo->ntables; + + pThreadInfo->lines = calloc(g_args.reqPerReq, sizeof(char *)); + if (NULL == pThreadInfo->lines) { + errorPrint2("Failed to alloc %"PRIu64" bytes, reason:%s\n", + g_args.reqPerReq * (uint64_t)sizeof(char *), + strerror(errno)); + return NULL; + } + + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { + if ((flagSleep) && (insert_interval)) { + st = taosGetTimestampMs(); + flagSleep = false; + } + + // generate data + + uint32_t recOfBatch = 0; + + for (uint64_t i = 0; i < batchPerTblTimes; i++) { + int64_t timestamp = startTime; + for (int j = recOfBatch; j < recOfBatch + batchPerTbl; j++) { + pThreadInfo->lines[j] = calloc(BUFFER_SIZE, 1); + if (NULL == pThreadInfo->lines[j]) { + errorPrint2("Failed to alloc %d bytes, reason:%s\n", + BUFFER_SIZE, strerror(errno)); + } + generateSmlTail(pThreadInfo->lines[j], *((char **)smlHeadList + i * sizeof(char *)), stbInfo, pThreadInfo, timestamp); + timestamp += timeStampStep; + } + tableSeq ++; + recOfBatch += batchPerTbl; + + pThreadInfo->totalInsertRows += batchPerTbl; + + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl, recOfBatch); + + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + // turn to first table + tableSeq = pThreadInfo->start_table_from; + generatedRecPerTbl += batchPerTbl; + + startTime = pThreadInfo->start_time + + generatedRecPerTbl * timeStampStep; + + flagSleep = true; + if (generatedRecPerTbl >= insertRows) + break; + + int64_t remainRows = insertRows - generatedRecPerTbl; + if ((remainRows > 0) && (batchPerTbl > remainRows)) + batchPerTbl = remainRows; + + if (pThreadInfo->ntables * batchPerTbl < g_args.reqPerReq) + break; + } + + verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n", + pThreadInfo->threadID, __func__, __LINE__, + generatedRecPerTbl, insertRows); + + if ((g_args.reqPerReq - recOfBatch) < batchPerTbl) + break; + } + + verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRIu64"\n", + pThreadInfo->threadID, __func__, __LINE__, recOfBatch, + pThreadInfo->totalInsertRows); + verbosePrint("[%d] %s() LN%d, buffer=%s\n", + pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer); + + startTs = taosGetTimestampUs(); + + if (recOfBatch == 0) { + errorPrint2("[%d] %s() LN%d Failed to insert records of batch %d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl); + if (batchPerTbl > 0) { + errorPrint("\tIf the batch is %d, the length of the SQL to insert a row must be less then %"PRId64"\n", + batchPerTbl, maxSqlLen / batchPerTbl); + } + errorPrint("\tPlease check if the buffer length(%"PRId64") or batch(%d) is set with proper value!\n", + maxSqlLen, batchPerTbl); + goto free_of_interlace; + } + int64_t affectedRows = execInsert(pThreadInfo, recOfBatch); + + endTs = taosGetTimestampUs(); + uint64_t delay = endTs - startTs; + performancePrint("%s() LN%d, insert execution time is %10.2f ms\n", + __func__, __LINE__, delay / 1000.0); + verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n", + pThreadInfo->threadID, + __func__, __LINE__, affectedRows); + + if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; + if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; + + if (recOfBatch != affectedRows) { + errorPrint2("[%d] %s() LN%d execInsert insert %d, affected rows: %"PRId64"\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, + recOfBatch, affectedRows, pThreadInfo->buffer); + goto free_of_interlace; + } + + pThreadInfo->totalAffectedRows += affectedRows; + + int currentPercent = pThreadInfo->totalAffectedRows * 100 / totalRows; + if (currentPercent > percentComplete ) { + printf("[%d]:%d%%\n", pThreadInfo->threadID, currentPercent); + percentComplete = currentPercent; + } + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + lastPrintTime = currentPrintTime; + } + + if ((insert_interval) && flagSleep) { + et = taosGetTimestampMs(); + + if (insert_interval > (et - st) ) { + uint64_t sleepTime = insert_interval - (et -st); + performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n", + __func__, __LINE__, sleepTime); + taosMsleep(sleepTime); // ms + sleepTimeTotal += insert_interval; + } + } + for (int index = 0; index < g_args.reqPerReq; index++) { + free(pThreadInfo->lines[index]); + } + } + if (percentComplete < 100) + printf("[%d]:%d%%\n", pThreadInfo->threadID, percentComplete); + +free_of_interlace: + tmfree(pThreadInfo->lines); + for (int index = 0; index < pThreadInfo->ntables; index++) { + tmfree(*(smlHeadList + index*(sizeof(char*)))); + } + tmfree(smlHeadList); + printStatPerThread(pThreadInfo); + return NULL; +} + // sync write interlace data static void* syncWriteInterlace(threadInfo *pThreadInfo, uint32_t interlaceRows) { debugPrint("[%d] %s() LN%d: ### interlace write\n", @@ -9947,6 +10455,121 @@ free_of_stmt_progressive: printStatPerThread(pThreadInfo); return NULL; } + +static void* syncWriteProgressiveSml(threadInfo *pThreadInfo) { + debugPrint("%s() LN%d: ### sml progressive write\n", __func__, __LINE__); + + SSuperTable* stbInfo = pThreadInfo->stbInfo; + int64_t timeStampStep = stbInfo->timeStampStep; + int64_t insertRows = stbInfo->insertRows; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", + __func__, __LINE__, insertRows); + + uint64_t lastPrintTime = taosGetTimestampMs(); + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + pThreadInfo->samplePos = 0; + + char *smlHeadList = calloc(pThreadInfo->ntables, sizeof(char *)); + assert(smlHeadList); + for (int t = 0; t < pThreadInfo->ntables; t++) { + char* smlHead = *((char**)smlHeadList + t * sizeof(char *)); + smlHead = (char *)calloc(HEAD_BUFF_LEN, 1); + if (NULL == smlHead) { + errorPrint2("calloc failed! size:%d\n", HEAD_BUFF_LEN); + exit(EXIT_FAILURE); + } + generateSmlHead(smlHead, stbInfo, pThreadInfo, t); + + } + int currentPercent = 0; + int percentComplete = 0; + + if (insertRows < g_args.reqPerReq) { + g_args.reqPerReq = insertRows; + } + pThreadInfo->lines = calloc(g_args.reqPerReq, sizeof(char *)); + if (NULL == pThreadInfo->lines) { + errorPrint2("Failed to alloc %"PRIu64" bytes, reason:%s\n", + g_args.reqPerReq * (uint64_t)sizeof(char *), + strerror(errno)); + return NULL; + } + + for (uint64_t i = 0; i < pThreadInfo->ntables; i++) { + int64_t timestamp = pThreadInfo->start_time; + + for (uint64_t j = 0; j < insertRows;) { + for (int k = 0; k < g_args.reqPerReq; k++) { + pThreadInfo->lines[k] = calloc(BUFFER_SIZE, 1); + if (NULL == pThreadInfo->lines[k]) { + errorPrint2("Failed to alloc %d bytes, reason:%s\n", + BUFFER_SIZE, strerror(errno)); + } + generateSmlTail(pThreadInfo->lines[k], *((char**)smlHeadList + i * sizeof(char *)), stbInfo, pThreadInfo, timestamp); + timestamp += timeStampStep; + j++; + if (j == insertRows) { + break; + } + } + uint64_t startTs = taosGetTimestampUs(); + int32_t affectedRows = execInsert(pThreadInfo, g_args.reqPerReq); + uint64_t endTs = taosGetTimestampUs(); + uint64_t delay = endTs - startTs; + + performancePrint("%s() LN%d, insert execution time is %10.f ms\n", + __func__, __LINE__, delay/1000.0); + verbosePrint("[%d] %s() LN%d affectedRows=%d\n", + pThreadInfo->threadID, + __func__, __LINE__, affectedRows); + + if (delay > pThreadInfo->maxDelay){ + pThreadInfo->maxDelay = delay; + } + if (delay < pThreadInfo->minDelay){ + pThreadInfo->minDelay = delay; + } + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; + + pThreadInfo->totalAffectedRows += affectedRows; + pThreadInfo->totalInsertRows += g_args.reqPerReq; + currentPercent = + pThreadInfo->totalAffectedRows * g_Dbs.threadCount / insertRows; + if (currentPercent > percentComplete) { + printf("[%d]:%d%%\n", pThreadInfo->threadID, + currentPercent); + percentComplete = currentPercent; + } + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + lastPrintTime = currentPrintTime; + } + + for (int index = 0; index < g_args.reqPerReq; index++) { + free(pThreadInfo->lines[index]); + } + if (j == insertRows) { + break; + } + } + } + tmfree(pThreadInfo->lines); + for (int index = 0; index < pThreadInfo->ntables; index++) { + free(*((char**)smlHeadList + index * sizeof(char *))); + } + tmfree(smlHeadList); + return NULL; +} + // sync insertion progressive data static void* syncWriteProgressive(threadInfo *pThreadInfo) { debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); @@ -10152,6 +10775,8 @@ static void* syncWrite(void *sarg) { #else return syncWriteInterlaceStmt(pThreadInfo, interlaceRows); #endif + } else if (SML_IFACE == stbInfo->iface) { + return syncWriteInterlaceSml(pThreadInfo, interlaceRows); } else { return syncWriteInterlace(pThreadInfo, interlaceRows); } @@ -10161,6 +10786,9 @@ static void* syncWrite(void *sarg) { if (((stbInfo) && (STMT_IFACE == stbInfo->iface)) || (STMT_IFACE == g_args.iface)) { return syncWriteProgressiveStmt(pThreadInfo); + } else if (((stbInfo) && (SML_IFACE == stbInfo->iface)) + || (SML_IFACE == g_args.iface)) { + return syncWriteProgressiveSml(pThreadInfo); } else { return syncWriteProgressive(pThreadInfo); } @@ -10318,7 +10946,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, // read sample data from file first int ret; - if (stbInfo) { + if (stbInfo && stbInfo->iface != SML_IFACE) { ret = prepareSampleForStb(stbInfo); } else { ret = prepareSampleForNtb(); @@ -10343,70 +10971,74 @@ static void startMultiThreadInsertData(int threads, char* db_name, uint64_t tableFrom; if (stbInfo) { - int64_t limit; - uint64_t offset; + if (stbInfo->iface != SML_IFACE) { + int64_t limit; + uint64_t offset; - if ((NULL != g_args.sqlFile) - && (stbInfo->childTblExists == TBL_NO_EXISTS) - && ((stbInfo->childTblOffset != 0) - || (stbInfo->childTblLimit >= 0))) { - printf("WARNING: offset and limit will not be used since the child tables not exists!\n"); - } + if ((NULL != g_args.sqlFile) + && (stbInfo->childTblExists == TBL_NO_EXISTS) + && ((stbInfo->childTblOffset != 0) + || (stbInfo->childTblLimit >= 0))) { + printf("WARNING: offset and limit will not be used since the child tables not exists!\n"); + } - if (stbInfo->childTblExists == TBL_ALREADY_EXISTS) { - if ((stbInfo->childTblLimit < 0) - || ((stbInfo->childTblOffset - + stbInfo->childTblLimit) - > (stbInfo->childTblCount))) { + if (stbInfo->childTblExists == TBL_ALREADY_EXISTS) { + if ((stbInfo->childTblLimit < 0) + || ((stbInfo->childTblOffset + + stbInfo->childTblLimit) + > (stbInfo->childTblCount))) { - if (stbInfo->childTblCount < stbInfo->childTblOffset) { - printf("WARNING: offset will not be used since the child tables count is less then offset!\n"); + if (stbInfo->childTblCount < stbInfo->childTblOffset) { + printf("WARNING: offset will not be used since the child tables count is less then offset!\n"); - stbInfo->childTblOffset = 0; + stbInfo->childTblOffset = 0; + } + stbInfo->childTblLimit = + stbInfo->childTblCount - stbInfo->childTblOffset; } - stbInfo->childTblLimit = - stbInfo->childTblCount - stbInfo->childTblOffset; + + offset = stbInfo->childTblOffset; + limit = stbInfo->childTblLimit; + } else { + limit = stbInfo->childTblCount; + offset = 0; } - offset = stbInfo->childTblOffset; - limit = stbInfo->childTblLimit; - } else { - limit = stbInfo->childTblCount; - offset = 0; - } + ntables = limit; + tableFrom = offset; - ntables = limit; - tableFrom = offset; + if ((stbInfo->childTblExists != TBL_NO_EXISTS) + && ((stbInfo->childTblOffset + stbInfo->childTblLimit) + > stbInfo->childTblCount)) { + printf("WARNING: specified offset + limit > child table count!\n"); + prompt(); + } - if ((stbInfo->childTblExists != TBL_NO_EXISTS) - && ((stbInfo->childTblOffset + stbInfo->childTblLimit) - > stbInfo->childTblCount)) { - printf("WARNING: specified offset + limit > child table count!\n"); - prompt(); - } + if ((stbInfo->childTblExists != TBL_NO_EXISTS) + && (0 == stbInfo->childTblLimit)) { + printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); + prompt(); + } - if ((stbInfo->childTblExists != TBL_NO_EXISTS) - && (0 == stbInfo->childTblLimit)) { - printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); - prompt(); - } + stbInfo->childTblName = (char*)calloc(1, + limit * TSDB_TABLE_NAME_LEN); + if (stbInfo->childTblName == NULL) { + taos_close(taos0); + errorPrint2("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + exit(EXIT_FAILURE); + } - stbInfo->childTblName = (char*)calloc(1, - limit * TSDB_TABLE_NAME_LEN); - if (stbInfo->childTblName == NULL) { - taos_close(taos0); - errorPrint2("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); - exit(EXIT_FAILURE); + int64_t childTblCount; + getChildNameOfSuperTableWithLimitAndOffset( + taos0, + db_name, stbInfo->stbName, + &stbInfo->childTblName, &childTblCount, + limit, + offset, stbInfo->escapeChar); + ntables = childTblCount; + } else { + ntables = stbInfo->childTblCount; } - - int64_t childTblCount; - getChildNameOfSuperTableWithLimitAndOffset( - taos0, - db_name, stbInfo->stbName, - &stbInfo->childTblName, &childTblCount, - limit, - offset, stbInfo->escapeChar); - ntables = childTblCount; } else { ntables = g_args.ntables; tableFrom = 0; @@ -10579,7 +11211,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint(); } */ - + if (g_args.iface == REST_IFACE || ((stbInfo) && (stbInfo->iface == REST_IFACE))) { #ifdef WINDOWS WSADATA wsaData; @@ -10604,7 +11236,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } pThreadInfo->sockfd = sockfd; } - + tsem_init(&(pThreadInfo->lock_sem), 0, 0); if (ASYNC_MODE == g_Dbs.asyncMode) { @@ -10990,33 +11622,34 @@ static int insertTestProcess() { double start; double end; - if (g_totalChildTables > 0) { - fprintf(stderr, - "creating %"PRId64" table(s) with %d thread(s)\n\n", - g_totalChildTables, g_Dbs.threadCountForCreateTbl); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "creating %"PRId64" table(s) with %d thread(s)\n\n", - g_totalChildTables, g_Dbs.threadCountForCreateTbl); - } + if (g_args.iface != SML_IFACE) { + if (g_totalChildTables > 0) { + fprintf(stderr, + "creating %"PRId64" table(s) with %d thread(s)\n\n", + g_totalChildTables, g_Dbs.threadCountForCreateTbl); + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "creating %"PRId64" table(s) with %d thread(s)\n\n", + g_totalChildTables, g_Dbs.threadCountForCreateTbl); + } - // create child tables - start = taosGetTimestampMs(); - createChildTables(); - end = taosGetTimestampMs(); + // create child tables + start = taosGetTimestampMs(); + createChildTables(); + end = taosGetTimestampMs(); - fprintf(stderr, - "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", - (end - start)/1000.0, g_totalChildTables, - g_Dbs.threadCountForCreateTbl, g_actualChildTables); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, - "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", - (end - start)/1000.0, g_totalChildTables, - g_Dbs.threadCountForCreateTbl, g_actualChildTables); + fprintf(stderr, + "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", + (end - start)/1000.0, g_totalChildTables, + g_Dbs.threadCountForCreateTbl, g_actualChildTables); + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, + "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", + (end - start)/1000.0, g_totalChildTables, + g_Dbs.threadCountForCreateTbl, g_actualChildTables); + } } } - // create sub threads for inserting data //start = taosGetTimestampMs(); for (int i = 0; i < g_Dbs.dbCount; i++) { @@ -11036,11 +11669,16 @@ static int insertTestProcess() { } } } else { - startMultiThreadInsertData( + if (SML_IFACE == g_args.iface) { + errorPrint2("%s\n", "Schemaless insertion must include stable"); + exit(EXIT_FAILURE); + } else { + startMultiThreadInsertData( g_Dbs.threadCount, g_Dbs.db[i].dbName, g_Dbs.db[i].dbCfg.precision, NULL); + } } } //end = taosGetTimestampMs(); @@ -12069,10 +12707,12 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", min(DATATYPE_BUFF_LEN, strlen("INT") + 1)); + g_Dbs.db[0].superTbls[0].tags[0].data_type = TSDB_DATA_TYPE_INT; g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", min(DATATYPE_BUFF_LEN, strlen("BINARY") + 1)); + g_Dbs.db[0].superTbls[0].tags[1].data_type = TSDB_DATA_TYPE_BINARY; g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.binwidth; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 34d9956c642c5c91aa72c6a4386dc675ad48dc98..69ec2968218a9e5b2ca34551c60b6c44256298d2 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -2091,7 +2091,7 @@ static int getTableDes( memset(tableDes->cols[i].value, 0, sizeof(tableDes->cols[i].note)); char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' convertNCharToReadable((char *)row[TSDB_SHOW_TABLES_NAME_INDEX], length[0], tbuf, COL_NOTE_LEN); - sprintf(tableDes->cols[i].value, "\'%s\'", tbuf); + sprintf(tableDes->cols[i].value, "%s", tbuf); break; } case TSDB_DATA_TYPE_TIMESTAMP: @@ -3005,7 +3005,13 @@ int main(int argc, char *argv[]) { printf("debug_print: %d\n", g_args.debug_print); for (int32_t i = 0; i < g_args.arg_list_len; i++) { - printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); + if (g_args.databases || g_args.all_databases) { + errorPrint("%s is an invalid input if database(s) be already specified.\n", + g_args.arg_list[i]); + exit(EXIT_FAILURE); + } else { + printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); + } } printf("==============================\n"); diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 83e54b97965f9dd0f8c1b484979cfd2e8919dbb1..4cf444bab2f05816c1af55d96156334800d758d5 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -32,7 +32,7 @@ ELSE () MESSAGE("") MESSAGE("${Green} use blm3 as httpd ${ColourReset}") EXECUTE_PROCESS( - COMMAND cd blm3 + COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR}/blm3 ) EXECUTE_PROCESS( COMMAND git rev-parse --short HEAD diff --git a/src/plugins/blm3 b/src/plugins/blm3 index 4bfae86dcabea0d5a40ff81a72be7c822737269b..f56aa0f485d7bb6aebbcefc2007eeecdccb767c8 160000 --- a/src/plugins/blm3 +++ b/src/plugins/blm3 @@ -1 +1 @@ -Subproject commit 4bfae86dcabea0d5a40ff81a72be7c822737269b +Subproject commit f56aa0f485d7bb6aebbcefc2007eeecdccb767c8 diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 107d3be228685be9ecd92125f226749a8cf20588..fea793fa860fd17ff30bcecae1436180bc6b34bf 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -204,7 +204,6 @@ static void monBuildMonitorSql(char *sql, int32_t cmd) { ", disk_used float, disk_total int" ", band_speed float" ", io_read float, io_write float" - ", io_read_rate float, io_write_rate float" ", req_http int, req_select int, req_insert int" ") tags (dnodeid int, fqdn binary(%d))", tsMonitorDbName, TSDB_FQDN_LEN); @@ -326,10 +325,7 @@ static int32_t monBuildIoSql(char *sql) { monDebug("failed to get io info"); } - float readRate = readKB/tsMonitorInterval; - float writeRate = writeKB/tsMonitorInterval; - - return sprintf(sql, ", %f, %f, %f, %f", readKB, writeKB, readRate, writeRate); + return sprintf(sql, ", %f, %f", readKB, writeKB); } static void monSaveSystemInfo() { diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index 548b03e1108f87feac0af5f93be69d4ee5569477..8bcd8fea42aa46ad3dd2f0857d88a9b4bb76dd4d 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -233,6 +233,7 @@ int32_t isValidFunction(const char* name, int32_t len); #define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) #define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) #define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) +#define IS_SCALAR_FUNCTION(x) (((x)&TSDB_FUNCSTATE_SCALAR) != 0) // determine the real data need to calculated the result enum { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 82f4f34a57c7d6d10a021fb2e426ff83cb3604e6..1bbc8a363ba3af678b42530db48eebb92deece48 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -312,7 +312,8 @@ typedef struct SQueryRuntimeEnv { STableQueryInfo *current; SRspResultInfo resultInfo; SHashObj *pTableRetrieveTsMap; - SUdfInfo *pUdfInfo; + SUdfInfo *pUdfInfo; + bool udfIsCopy; } SQueryRuntimeEnv; enum { diff --git a/src/query/inc/qUdf.h b/src/query/inc/qUdf.h index 1083b1e698f7591aae4586c7722e5343cd9c4d86..77da4b668ae08d10a6154cdece59ec62f5114be9 100644 --- a/src/query/inc/qUdf.h +++ b/src/query/inc/qUdf.h @@ -51,6 +51,7 @@ typedef struct SUdfInfo { SUdfInit init; char *content; char *path; + bool keep; } SUdfInfo; //script diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c773efd9971009f47eee24a434365c3b6ef0db07..9457483714310c9ada01148345927f44718a28e8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -988,13 +988,12 @@ void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); + (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, (int32_t *)&pCtx->resultInfo->numOfRes); } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); + (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, (int32_t *)&pCtx->resultInfo->numOfRes, &pUdfInfo->init); } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { + + if (pCtx->resultInfo->numOfRes > 0) { pCtx->resultInfo->hasResult = DATA_SET_FLAG; } @@ -2409,8 +2408,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->sasArray); } - destroyUdfInfo(pRuntimeEnv->pUdfInfo); - + if (!pRuntimeEnv->udfIsCopy) { + destroyUdfInfo(pRuntimeEnv->pUdfInfo); + } + destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); @@ -8039,7 +8040,7 @@ static char* getUdfFuncName(char* funcname, char* name, int type) { } int32_t initUdfInfo(SUdfInfo* pUdfInfo) { - if (pUdfInfo == NULL) { + if (pUdfInfo == NULL || pUdfInfo->handle) { return TSDB_CODE_SUCCESS; } //qError("script len: %d", pUdfInfo->contLen); @@ -8074,10 +8075,21 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { // TODO check for failure of flush to disk /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); fclose(file); - tfree(pUdfInfo->content); + if (!pUdfInfo->keep) { + tfree(pUdfInfo->content); + } + if (pUdfInfo->path) { + unlink(pUdfInfo->path); + } + + tfree(pUdfInfo->path); pUdfInfo->path = strdup(path); + if (pUdfInfo->handle) { + taosCloseDll(pUdfInfo->handle); + } + pUdfInfo->handle = taosLoadDll(path); if (NULL == pUdfInfo->handle) { @@ -8092,9 +8104,17 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); + pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); - pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] || NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE]) { + return TSDB_CODE_QRY_SYS_ERROR; + } + } else { + if (pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] || pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE]) { + return TSDB_CODE_QRY_SYS_ERROR; + } } pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); @@ -8201,7 +8221,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp } int32_t param = (int32_t)pExprs[i].base.param[0].i64; - if (pExprs[i].base.functionId != TSDB_FUNC_ARITHM && + if (pExprs[i].base.functionId > 0 && pExprs[i].base.functionId != TSDB_FUNC_ARITHM && (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { tfree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; diff --git a/src/tsdb/inc/tsdbReadImpl.h b/src/tsdb/inc/tsdbReadImpl.h index f067905f5c0dcf4fde64dfc066d4d1a1dbabb25d..20d8b88c839411136793556de55450577ffecaec 100644 --- a/src/tsdb/inc/tsdbReadImpl.h +++ b/src/tsdb/inc/tsdbReadImpl.h @@ -71,11 +71,10 @@ typedef struct { * blkVer; // 0 - original block, 1 - block since importing .smad/.smal * aggrOffset; // only valid when blkVer > 0 and aggrStat > 0 */ -#define SBlockFieldsP1 \ - uint64_t aggrStat : 3; \ - uint64_t blkVer : 5; \ - uint64_t aggrOffset : 56; \ - uint32_t aggrLen +#define SBlockFieldsP1 \ + uint64_t aggrStat : 1; \ + uint64_t blkVer : 7; \ + uint64_t aggrOffset : 56 typedef struct { SBlockFieldsP0; @@ -125,7 +124,6 @@ typedef struct { int32_t len; uint32_t type : 8; uint32_t offset : 24; - // char padding[]; } SBlockColV1; #define SBlockCol SBlockColV1 // latest SBlockCol definition @@ -160,10 +158,8 @@ typedef struct { uint64_t uid; // For recovery usage SBlockCol cols[]; } SBlockData; -typedef struct { - int32_t numOfCols; // For recovery usage - SAggrBlkCol cols[]; -} SAggrBlkData; + +typedef void SAggrBlkData; // SBlockCol cols[]; struct SReadH { STsdbRepo * pRepo; @@ -207,8 +203,7 @@ static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { } } -#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) \ - (sizeof(SAggrBlkData) + sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) +#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) { switch (blkVer) { diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 53ca51004fc406b05028d13be7cd2bf054771c0d..eccdcbdbf6155ca37dc35bc171189de3781e8642 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -1080,11 +1080,10 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDFile // Get # of cols not all NULL(not including key column) int nColsNotAllNull = 0; - int nAggrCols = 0; for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol * pBlockCol = pBlockData->cols + nColsNotAllNull; - SAggrBlkCol *pAggrBlkCol = pAggrBlkData->cols + nColsNotAllNull; + SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull; if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; @@ -1106,7 +1105,6 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDFile (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), &(pAggrBlkCol->numOfNull)); - ++nAggrCols; } nColsNotAllNull++; } @@ -1188,9 +1186,8 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDFile return -1; } - uint32_t aggrStatus = ((nAggrCols > 0) && (rowsToWrite > 8)) ? 1 : 0; // TODO: How to make the decision? + uint32_t aggrStatus = ((nColsNotAllNull > 0) && (rowsToWrite > 8)) ? 1 : 0; // TODO: How to make the decision? if (aggrStatus > 0) { - pAggrBlkData->numOfCols = nColsNotAllNull; taosCalcChecksumAppend(0, (uint8_t *)pAggrBlkData, tsizeAggr); tsdbUpdateDFileMagic(pDFileAggr, POINTER_SHIFT(pAggrBlkData, tsizeAggr - sizeof(TSCKSUM))); @@ -1216,7 +1213,6 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDFile pBlock->aggrStat = aggrStatus; pBlock->blkVer = SBlockVerLatest; pBlock->aggrOffset = (uint64_t)offsetAggr; - pBlock->aggrLen = tsizeAggr; tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 6415c9a649ec08e4db442a321565e15e5df2e4a7..3b5e8ce56dab297c5b6cc4a9b07d8150445917b9 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -441,6 +441,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { if ((tdInitDataCols(pComph->pDataCols, pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0)) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tdFreeSchema(pSchema); return -1; } tdFreeSchema(pSchema); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 62160c2f36762e1c55e858a7026360b8287f6c3f..2dea0de055c3f71d3a13b5ecb3c92b878f25d1e6 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -185,6 +185,14 @@ int tsdbUnlockRepo(STsdbRepo *pRepo) { return 0; } +int tsdbCheckWal(STsdbRepo *pRepo, uint32_t walSize) { // MB + STsdbCfg *pCfg = &(pRepo->config); + if ((walSize > tsdbWalFlushSize) && (walSize > (pCfg->totalBlocks / 2 * pCfg->cacheBlockSize))) { + if (tsdbAsyncCommit(pRepo) < 0) return -1; + } + return 0; +} + int tsdbCheckCommit(STsdbRepo *pRepo) { ASSERT(pRepo->mem != NULL); STsdbCfg *pCfg = &(pRepo->config); @@ -196,7 +204,6 @@ int tsdbCheckCommit(STsdbRepo *pRepo) { // trigger commit if (tsdbAsyncCommit(pRepo) < 0) return -1; } - return 0; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ca6912cdd887a1fdb200bee82cbf542c517f08a3..5e4ab00b4158d2f1c15b3fe47e3a296ff429edfa 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1620,7 +1620,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, SColIdx *pColIdx = kvRowColIdxAt(rowBody, chosen_itr); colId = pColIdx->colId; offset = pColIdx->offset; - value = tdGetKvRowDataOfCol(rowBody, pColIdx->offset); + value = tdGetKvRowDataOfCol(rowBody, offset); } diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index e55944d5bb83ca614da862f9c350a41690b0ca6e..4976e8b8fb8213b6a9cdedbf380d812e117f1fc8 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -580,12 +580,12 @@ void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBl SAggrBlkData *pAggrBlkData = pReadh->pAggrBlkData; for (int i = 0, j = 0; i < numOfCols;) { - if (j >= pAggrBlkData->numOfCols) { + if (j >= pBlock->numOfCols) { pStatis[i].numOfNull = -1; i++; continue; } - SAggrBlkCol *pAggrBlkCol = ((SAggrBlkCol *)(pAggrBlkData->cols)) + j; + SAggrBlkCol *pAggrBlkCol = ((SAggrBlkCol *)(pAggrBlkData)) + j; if (pStatis[i].colId == pAggrBlkCol->colId) { pStatis[i].sum = pAggrBlkCol->sum; pStatis[i].max = pAggrBlkCol->max; diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 2ba4b964c04b0a1ca9f883cd619aae2b7fcbe1d7..3ba0031a07a83d7374b3f940cf51bf31478cfd38 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 124 +#define TSDB_CFG_MAX_NUM 125 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 066040170e44c24539e29b5de5acd438e8b9b9d0..11a04cdf9480927131a92753a56bb67b462500ab 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -37,6 +37,7 @@ int32_t tfFsync(int64_t tfd); bool tfValid(int64_t tfd); int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); int32_t tfFtruncate(int64_t tfd, int64_t length); +int32_t tfStat(int64_t tfd, struct stat *pFstat); #ifdef __cplusplus } diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 455c885e753e35724d27ec223f86ebf04751286f..d975995b2149297d2ae53584bcf36b6c74b7b529 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -133,3 +133,14 @@ int32_t tfFtruncate(int64_t tfd, int64_t length) { taosReleaseRef(tsFileRsetId, tfd); return code; } + +int32_t tfStat(int64_t tfd, struct stat *pFstat) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + int32_t code = fstat(fd, pFstat); + + taosReleaseRef(tsFileRsetId, tfd); + return code; +} diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c index 99ade6db1a76fdac48e441a2c1a9a2a3d388f812..8414b6292c43f674d62a61338ca271273922a209 100644 --- a/src/util/src/ttokenizer.c +++ b/src/util/src/ttokenizer.c @@ -628,7 +628,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { t0.n = 0; return t0; } - + t = str[++(*i)]; } diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 35ec90cb2efbb270c8b007f9bdb347333a87fded..31556b83d06224d285db29650eba82c4d3acab5e 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -53,13 +53,13 @@ int32_t strdequote(char *z) { } -int32_t strRmquote(char *z, int32_t len){ +int32_t strRmquote(char *z, int32_t len){ // delete escape character: \\, \', \" char delim = z[0]; if (delim != '\'' && delim != '\"') { return len; } - + int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < len - 1; ++k) { @@ -74,23 +74,24 @@ int32_t strRmquote(char *z, int32_t len){ continue; } } - + z[j] = z[k]; j++; } - + z[j] = 0; - + return len - 2 - cnt; } int32_t strRmquoteEscape(char *z, int32_t len) { if (len <= 0) return len; - + if (z[0] == '\'' || z[0] == '\"') { return strRmquote(z, len); } else if (len > 1 && z[0] == TS_ESCAPE_CHAR && z[len - 1] == TS_ESCAPE_CHAR) { memmove(z, z + 1, len - 2); + z[len - 2] = '\0'; return len - 2; } diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 4864b79dc4ee7c718ad7c023277793f21e74446d..1deceebb0ad3bd146a3cd81fab4cabb2d290b037 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -56,6 +56,7 @@ typedef struct { uint64_t version; // current version uint64_t cversion; // version while commit start uint64_t fversion; // version on saved data file + uint32_t tblMsgVer; // create table msg version void * wqueue; // write queue void * qqueue; // read query queue void * fqueue; // read fetch/cancel queue diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index e8ac978bb2d163ff0a8eda78015efae9f817ac34..35c2ab72dfdf55f66b1095c757fc4f90656c842b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -36,6 +36,7 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite); +static int32_t vnodeCheckWal(SVnodeObj *pVnode); int32_t vnodeInitWrite(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessSubmitMsg; @@ -167,6 +168,13 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR return code; } +static int32_t vnodeCheckWal(SVnodeObj *pVnode) { + if (pVnode->isCommiting == 0) { + return tsdbCheckWal(pVnode->tsdb, (uint32_t)(walGetFSize(pVnode->wal) >> 20)); + } + return 0; +} + static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { int code = TSDB_CODE_SUCCESS; @@ -181,6 +189,10 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe ASSERT(code != 0); } + if (((++pVnode->tblMsgVer) & 32767) == 0) { // lazy check + vnodeCheckWal(pVnode); + } + tsdbClearTableCfg(pCfg); return code; } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e991bf02aa68c92d7cf4dfdb09982ebaa6541bdc..3f2df3f6243c3291602ceb0c7fcd93d475927485 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -576,4 +576,14 @@ void walResetVersion(twalh param, uint64_t newVer) { wInfo("vgId:%d, version reset from %" PRIu64 " to %" PRIu64, pWal->vgId, pWal->version, newVer); pWal->version = newVer; +} + +int64_t walGetFSize(twalh handle) { + SWal *pWal = handle; + if (pWal == NULL) return 0; + struct stat _fstat; + if (tfStat(pWal->tfd, &_fstat) == 0) { + return _fstat.st_size; + }; + return 0; } \ No newline at end of file diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index d9d2a41cb2782f1e919857d3a94c5f83946bb277..2c197887e7d7f1e6397213641a02ee8b37a84190 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -15,7 +15,7 @@ static void prepare_data(TAOS* taos) { result = taos_query(taos, "drop database if exists test;"); taos_free_result(result); usleep(100000); - result = taos_query(taos, "create database test precision 'us';"); + result = taos_query(taos, "create database test precision 'ns';"); taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); @@ -288,12 +288,12 @@ void verify_stream(TAOS* taos) { taos_close_stream(strm); } -int32_t verify_schema_less(TAOS* taos) { +void verify_schema_less(TAOS* taos) { TAOS_RES* result; result = taos_query(taos, "drop database if exists test;"); taos_free_result(result); usleep(100000); - result = taos_query(taos, "create database test precision 'us' update 1;"); + result = taos_query(taos, "create database test precision 'ns' update 1 keep 36500;"); taos_free_result(result); usleep(100000); @@ -302,7 +302,7 @@ int32_t verify_schema_less(TAOS* taos) { taos_free_result(result); usleep(100000); - int code = 0; + int code = 0, affected_rows = 0; char* lines[] = { "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", @@ -316,41 +316,106 @@ int32_t verify_schema_less(TAOS* taos) { "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000" }; - code = taos_schemaless_insert(taos, lines , sizeof(lines)/sizeof(char*), 0, "ns"); + result = taos_schemaless_insert(taos, lines , sizeof(lines)/sizeof(char*), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines1]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); char* lines2[] = { "stg,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", "stg,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833641000000" }; - code = taos_schemaless_insert(taos, &lines2[0], 1, 0, "ns"); - code = taos_schemaless_insert(taos, &lines2[1], 1, 0, "ns"); + result = taos_schemaless_insert(taos, &lines2[0], 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines2_0]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); + + result = taos_schemaless_insert(taos, &lines2[1], 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines2_1]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); char* lines3[] = { "sth,t1=4i64,t2=5f64,t4=5f64,ID=childTable c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641", "sth,t1=4i64,t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933654" }; - code = taos_schemaless_insert(taos, lines3, 2, 0, "ms"); + result = taos_schemaless_insert(taos, lines3, 2, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines3]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); char* lines4[] = { "st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", "dgtyqodr,t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532" }; - code = taos_schemaless_insert(taos, lines4, 2, 0, "ns"); + result = taos_schemaless_insert(taos, lines4, 2, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines4]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); + char* lines5[] = { "zqlbgs,id=zqlbgs_39302_21680,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "zqlbgs,t9=f,id=zqlbgs_39302_21680,t0=f,t1=127i8,t11=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t10=L\"ncharTagValue\" c10=f,c0=f,c1=127i8,c12=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\" 1626006833639000000" }; - code = taos_schemaless_insert(taos, &lines5[0], 1, 0, "ns"); - code = taos_schemaless_insert(taos, &lines5[1], 1, 0, "ns"); + result = taos_schemaless_insert(taos, &lines5[0], 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines5_0]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); + + result = taos_schemaless_insert(taos, &lines5[1], 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines5_1]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); char* lines6[] = { "st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", "dgtyqodr,t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532" }; - code = taos_schemaless_insert(taos, lines6, 2, 0, "ns"); + result = taos_schemaless_insert(taos, lines6, 2, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines6]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", code, taos_errstr(result), affected_rows); + } + taos_free_result(result); + + //Test timestamp precision + char* lines7[] = { + "stts,t1=10i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1", + }; + + for (int precision = TSDB_SML_TIMESTAMP_HOURS; precision <= TSDB_SML_TIMESTAMP_NANO_SECONDS; ++precision) { + result = taos_schemaless_insert(taos, lines7, 1, TSDB_SML_LINE_PROTOCOL, precision); + code = taos_errno(result); + if (code != TSDB_CODE_SUCCESS) { + affected_rows = taos_affected_rows(result); + printf("\033[31m [lines7_%d]taos_schemaless_insert failed, code: %d,%s, affected rows:%d \033[0m\n", precision, code, taos_errstr(result), affected_rows); + } + taos_free_result(result); + } - return (code); } int main(int argc, char* argv[]) { diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c index 0fb8754de666d7067ef3dcbf9b7797592ca5b61b..05df33ffe6f0c08dd5608bb3ba30a21623f2ae45 100644 --- a/tests/examples/c/epoll.c +++ b/tests/examples/c/epoll.c @@ -92,7 +92,7 @@ static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client); fprintf(stderr, "" fmt "\n", ##__VA_ARGS__); \ } \ fprintf(stderr, "usage:\n"); \ - fprintf(stderr, " %s -l : specify listenning port\n", arg0); \ + fprintf(stderr, " %s -l : specify listening port\n", arg0); \ } while (0) int main(int argc, char *argv[]) { @@ -256,7 +256,7 @@ static int open_listen(unsigned short port) { E("getsockname() failed"); } A(len == sizeof(si), "internal logic error"); - D("listenning at: %d", ntohs(si.sin_port)); + D("listening at: %d", ntohs(si.sin_port)); return skt; } while (0); close(skt); diff --git a/tests/examples/c/schemaless.c b/tests/examples/c/schemaless.c index 0d98acb03a27cd3c72568d8f713cf392e5bd057c..9a2d2f063573d26093bd5032e2f68cc54fc5f908 100644 --- a/tests/examples/c/schemaless.c +++ b/tests/examples/c/schemaless.c @@ -33,6 +33,8 @@ typedef struct { int numBatches; SThreadLinesBatch batches[MAX_THREAD_LINE_BATCHES]; int64_t costTime; + int tsPrecision; + int lineProtocol; } SThreadInsertArgs; static void* insertLines(void* args) { @@ -43,10 +45,12 @@ static void* insertLines(void* args) { SThreadLinesBatch* batch = insertArgs->batches + i; printf("%s, thread: 0x%s\n", "begin taos_insert_lines", tidBuf); int64_t begin = getTimeInUs(); - int32_t code = taos_schemaless_insert(insertArgs->taos, batch->lines, batch->numLines, 0, "ms"); + TAOS_RES *res = taos_schemaless_insert(insertArgs->taos, batch->lines, batch->numLines, insertArgs->lineProtocol, insertArgs->tsPrecision); + int32_t code = taos_errno(res); int64_t end = getTimeInUs(); insertArgs->costTime += end - begin; - printf("code: %d, %s. time used:%"PRId64", thread: 0x%s\n", code, tstrerror(code), end - begin, tidBuf); + printf("code: %d, %s. affected lines:%d time used:%"PRId64", thread: 0x%s\n", code, taos_errstr(res), taos_affected_rows(res), end - begin, tidBuf); + taos_free_result(res); } return NULL; } @@ -95,9 +99,11 @@ int main(int argc, char* argv[]) { int numFields = 13; int maxLinesPerBatch = 16384; + int tsPrecision = TSDB_SML_TIMESTAMP_NOT_CONFIGURED; + int lineProtocol = TSDB_SML_UNKNOWN_PROTOCOL; int opt; - while ((opt = getopt(argc, argv, "s:c:r:f:t:m:h")) != -1) { + while ((opt = getopt(argc, argv, "s:c:r:f:t:m:p:P:h")) != -1) { switch (opt) { case 's': numSuperTables = atoi(optarg); @@ -117,6 +123,12 @@ int main(int argc, char* argv[]) { case 'm': maxLinesPerBatch = atoi(optarg); break; + case 'p': + tsPrecision = atoi(optarg); + break; + case 'P': + lineProtocol = atoi(optarg); + break; case 'h': fprintf(stderr, "Usage: %s -s supertable -c childtable -r rows -f fields -t threads -m maxlines_per_batch\n", argv[0]); @@ -178,6 +190,8 @@ int main(int argc, char* argv[]) { args.taos = taos; args.batches[0].lines = linesStb; args.batches[0].numLines = numSuperTables; + args.tsPrecision = tsPrecision; + args.lineProtocol = lineProtocol; insertLines(&args); for (int i = 0; i < numSuperTables; ++i) { free(linesStb[i]); diff --git a/tests/examples/python/taosdemo/taosdemo.py b/tests/examples/python/taosdemo/taosdemo.py index d55023bdbf119544a788aa6246c9d63dbf024872..4aaf00157c5fe5bbeec27b001f663a94c1d89439 100755 --- a/tests/examples/python/taosdemo/taosdemo.py +++ b/tests/examples/python/taosdemo/taosdemo.py @@ -21,78 +21,91 @@ import json import random import time import datetime +import multiprocessing from multiprocessing import Manager, Pool, Lock from multipledispatch import dispatch from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED @dispatch(str, str) -def v_print(msg: str, arg: str): +def v_print(msg, arg): + # type: (str, str) -> None if verbose: print(msg % arg) @dispatch(str, str, str) -def v_print(msg: str, arg1: str, arg2: str): +def v_print(msg, arg1, arg2): + # type: (str, str, str) -> None if verbose: print(msg % (arg1, arg2)) @dispatch(str, str, str, str) -def v_print(msg: str, arg1: str, arg2: str, arg3: str): +def v_print(msg, arg1, arg2, arg3): + # type: (str, str, str, str) -> None if verbose: print(msg % (arg1, arg2, arg3)) @dispatch(str, str, str, str, str) -def v_print(msg: str, arg1: str, arg2: str, arg3: str, arg4: str): +def v_print(msg, arg1, arg2, arg3, arg4): + # type: (str, str, str, str, str) -> None if verbose: print(msg % (arg1, arg2, arg3, arg4)) @dispatch(str, int) -def v_print(msg: str, arg: int): +def v_print(msg, arg): + # type: (str, int) -> None if verbose: print(msg % int(arg)) @dispatch(str, int, str) -def v_print(msg: str, arg1: int, arg2: str): +def v_print(msg, arg1, arg2): + # type: (str, int, str) -> None if verbose: print(msg % (int(arg1), str(arg2))) @dispatch(str, str, int) -def v_print(msg: str, arg1: str, arg2: int): +def v_print(msg, arg1, arg2): + # type: (str, str, int) -> None if verbose: print(msg % (arg1, int(arg2))) @dispatch(str, int, int) -def v_print(msg: str, arg1: int, arg2: int): +def v_print(msg, arg1, arg2): + # type: (str, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2))) @dispatch(str, int, int, str) -def v_print(msg: str, arg1: int, arg2: int, arg3: str): +def v_print(msg, arg1, arg2, arg3): + # type: (str, int, int, str) -> None if verbose: print(msg % (int(arg1), int(arg2), str(arg3))) @dispatch(str, int, int, int) -def v_print(msg: str, arg1: int, arg2: int, arg3: int): +def v_print(msg, arg1, arg2, arg3): + # type: (str, int, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2), int(arg3))) @dispatch(str, int, int, int, int) -def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): +def v_print(msg, arg1, arg2, arg3, arg4): + # type: (str, int, int, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) -def restful_execute(host: str, port: int, user: str, password: str, cmd: str): +def restful_execute(host, port, user, password, cmd): + # type: (str, int, str, str, str) -> None url = "http://%s:%d/rest/sql" % (host, restPort) v_print("restful_execute - cmd: %s", cmd) @@ -112,7 +125,8 @@ def restful_execute(host: str, port: int, user: str, password: str, cmd: str): print("resp: %s" % json.dumps(resp.json())) -def query_func(process: int, thread: int, cmd: str): +def query_func(process, thread, cmd): + # type: (int, int, str) -> None v_print("%d process %d thread cmd: %s", process, thread, cmd) if oneMoreHost != "NotSupported" and random.randint( @@ -133,7 +147,8 @@ def query_func(process: int, thread: int, cmd: str): host, port, user, password, cmd) -def query_data_process(cmd: str): +def query_data_process(cmd): + # type: (str) -> None # establish connection if native if native: v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) @@ -256,7 +271,8 @@ def drop_databases(): (dbName, i)) -def insert_func(process: int, thread: int): +def insert_func(process, thread): + # type: (int, int) -> None v_print("%d process %d thread, insert_func ", process, thread) # generate uuid @@ -374,7 +390,8 @@ def create_tb(): (tbName, j)) -def insert_data_process(lock, i: int, begin: int, end: int): +def insert_data_process(lock, i, begin, end): + # type: (multiprocessing._LockType, int, int, int) -> None lock.acquire() tasks = end - begin v_print("insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks) @@ -675,7 +692,10 @@ if __name__ == "__main__": printConfig() if not skipPrompt: - input("Press any key to continue..") + try: + input("Press any key to continue..") + except SyntaxError: + pass # establish connection first if native if native: diff --git a/tests/pytest/client/twoClients.py b/tests/pytest/client/twoClients.py index 358c4e851f7fa90caa8dd069e6b9b5064e44eb40..82191156bf425ca1f05e52516dbc04615255131c 100644 --- a/tests/pytest/client/twoClients.py +++ b/tests/pytest/client/twoClients.py @@ -36,7 +36,8 @@ class TwoClients: tdDnodes.deploy(1) tdDnodes.start(1) - # first client create a stable and insert data + tdLog.sleep(2) + # first client create a stable and insert data conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=tdDnodes.getSimCfgPath()) cursor1 = conn1.cursor() cursor1.execute("drop database if exists db") @@ -90,6 +91,8 @@ class TwoClients: cursor2.close() conn1.close() conn2.close() + + tdLog.success("%s successfully executed" % __file__) clients = TwoClients() clients.initConnection() diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 998109a99f659e44e3aaddb9f8f5e17474cb153f..183d100bc9365883416f5dfdd0008bd8a0c5dc30 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -313,7 +313,7 @@ python3 testNoCompress.py python3 testMinTablesPerVnode.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py -#python3 client/twoClients.py +python3 client/twoClients.py python3 test.py -f query/queryInterval.py python3 test.py -f query/queryFillTest.py # subscribe @@ -391,6 +391,7 @@ python3 ./test.py -f tag_lite/alter_tag.py python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py python3 test.py -f tools/taosdemoAllTest/TD-5213/insert4096columns_not_use_taosdemo.py python3 test.py -f tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py +python3 test.py -f tools/taosdemoAllTest/TD-10539/create_taosdemo.py python3 ./test.py -f tag_lite/drop_auto_create.py python3 test.py -f insert/insert_before_use_db.py python3 test.py -f alter/alter_keep.py diff --git a/tests/pytest/functions/queryTestCases.py b/tests/pytest/functions/queryTestCases.py index b254908c42ec8f267fa947b5c759101b16b066a9..e5213d9f7253d1a82e84584c87b66291daf01f25 100644 --- a/tests/pytest/functions/queryTestCases.py +++ b/tests/pytest/functions/queryTestCases.py @@ -418,10 +418,14 @@ class TDTestCase: tdSql.execute("use db") tdSql.execute("create stable db.stb1 (ts timestamp, c1 int) tags(t1 int)") + nowtime = int(round(time.time() * 1000)) for i in range(1000): tdSql.execute(f"create table db.t1{i} using db.stb1 tags({i})") + sql = f"insert into db.t1{i} values" for j in range(260): - tdSql.execute(f"insert into db.t1{i} values (now-100d, {i+j})") + sql += f"({nowtime-1000*i-j}, {i+j})" + # tdSql.execute(f"insert into db.t1{i} values (now-100d, {i+j})") + tdSql.execute(sql) # tdDnodes.stop(dnode_index) # tdDnodes.start(dnode_index) @@ -465,7 +469,7 @@ class TDTestCase: # tdSql.execute("insert into db.t1 values ('2021-07-01 08:00:04.000', 1001.51, 1001.52, 1001.53, 1001.54)") # for i in range(1000000): - for i in range(1000000): + for i in range(10000): random1 = random.uniform(1000,1001) random2 = random.uniform(1000,1001) random3 = random.uniform(1000,1001) diff --git a/tests/pytest/insert/insertJSONPayload.py b/tests/pytest/insert/insertJSONPayload.py index c5cd96f86d984bff09dcc3ee40405bf5c2056fea..81d4b47ef15cb03311943d3d53c2efe25a3b0312 100644 --- a/tests/pytest/insert/insertJSONPayload.py +++ b/tests/pytest/insert/insertJSONPayload.py @@ -15,6 +15,7 @@ import sys from util.log import * from util.cases import * from util.sql import * +from util.types import TDSmlProtocolType, TDSmlTimestampType class TDTestCase: @@ -35,7 +36,7 @@ class TDTestCase: print("============= step0 : test metric ================") payload = [''' { - "metric": "`.stb.0.`", + "metric": ".stb.0.", "timestamp": 1626006833610, "value": 10, "tags": { @@ -46,7 +47,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe `.stb.0.`") @@ -67,7 +68,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_0") @@ -86,7 +87,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_1") @@ -105,7 +106,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_2") @@ -124,7 +125,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_3") @@ -143,7 +144,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_4") @@ -162,7 +163,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_5") @@ -184,7 +185,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) ### timestamp 10 digits second ### @@ -201,7 +202,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) print("============= step3 : test tags ================") @@ -216,7 +217,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_8") @@ -232,7 +233,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_9") @@ -248,7 +249,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb0_10") @@ -274,7 +275,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select ts from stb1_0") @@ -297,7 +298,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select ts from stb1_1") @@ -320,7 +321,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select ts from stb1_2") @@ -343,7 +344,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select ts from stb1_3") @@ -367,7 +368,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) ### metric value ### @@ -390,7 +391,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_0") @@ -415,7 +416,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_1") @@ -440,7 +441,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_2") @@ -465,7 +466,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_3") @@ -490,7 +491,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_4") @@ -515,7 +516,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_5") @@ -540,7 +541,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_6") @@ -565,7 +566,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_7") @@ -590,7 +591,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb2_8") @@ -649,7 +650,7 @@ class TDTestCase: } } '''] - code = self._conn.schemaless_insert(payload, 2, None) + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("describe stb3_0") @@ -663,6 +664,183 @@ class TDTestCase: tdSql.checkData(9, 1, "BINARY") tdSql.checkData(10, 1, "NCHAR") + ### special characters ### + + payload = [''' + { + "metric": "1234", + "timestamp": 1626006833, + "value": 1, + "tags": { + "id": "123", + "456": true, + "int": false, + "double": 1, + "into": 1, + "from": 2, + "!@#$.%^&*()": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + '''] + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) + print("schemaless_insert result {}".format(code)) + + tdSql.query("describe `1234`") + tdSql.checkRows(8) + + tdSql.query("select * from `123`") + tdSql.checkRows(1) + + payload = [''' + { + "metric": "int", + "timestamp": 1626006833, + "value": 1, + "tags": { + "id": "and", + "456": true, + "int": false, + "double": 1, + "into": 1, + "from": 2, + "!@#$.%^&*()": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + '''] + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) + print("schemaless_insert result {}".format(code)) + + tdSql.query("describe `int`") + tdSql.checkRows(8) + + tdSql.query("select * from `and`") + tdSql.checkRows(1) + + payload = [''' + { + "metric": "double", + "timestamp": 1626006833, + "value": 1, + "tags": { + "id": "for", + "456": true, + "int": false, + "double": 1, + "into": 1, + "from": 2, + "!@#$.%^&*()": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + '''] + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) + print("schemaless_insert result {}".format(code)) + + tdSql.query("describe `double`") + tdSql.checkRows(8) + + tdSql.query("select * from `for`") + tdSql.checkRows(1) + + payload = [''' + { + "metric": "from", + "timestamp": 1626006833, + "value": 1, + "tags": { + "id": "!@#.^&", + "456": true, + "int": false, + "double": 1, + "into": 1, + "from": 2, + "!@#$.%^&*()": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + '''] + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) + print("schemaless_insert result {}".format(code)) + + tdSql.query("describe `from`") + tdSql.checkRows(8) + + tdSql.query("select * from `!@#.^&`") + tdSql.checkRows(1) + + payload = [''' + { + "metric": "!@#$.%^&*()", + "timestamp": 1626006833, + "value": 1, + "tags": { + "id": "none", + "456": true, + "int": false, + "double": 1, + "into": 1, + "from": 2, + "!@#$.%^&*()": "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>" + } + } + '''] + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) + print("schemaless_insert result {}".format(code)) + + tdSql.query("describe `!@#$.%^&*()`") + tdSql.checkRows(8) + + tdSql.query("select * from `none`") + tdSql.checkRows(1) + + payload = [''' + { + "metric": "STABLE", + "timestamp": { + "value": 1626006833, + "type": "s" + }, + "value": { + "value": "hello", + "type": "nchar" + }, + "tags": { + "id": "KEY", + "456": { + "value": true, + "type": "bool" + }, + "int": { + "value": 127, + "type": "tinyint" + }, + "double":{ + "value": 32767, + "type": "smallint" + }, + "into": { + "value": 2147483647, + "type": "int" + }, + "INSERT": { + "value": 9.2233720368547758e+18, + "type": "bigint" + }, + "!@#$.%^&*()": { + "value": 11.12345, + "type": "float" + } + } + } + '''] + + code = self._conn.schemaless_insert(payload, TDSmlProtocolType.JSON.value, TDSmlTimestampType.NOT_CONFIGURED.value) + print("schemaless_insert result {}".format(code)) + + tdSql.query("describe `stable`") + tdSql.checkRows(8) + + tdSql.query("select * from `key`") + tdSql.checkRows(1) + def stop(self): tdSql.close() diff --git a/tests/pytest/insert/insertTelnetLines.py b/tests/pytest/insert/insertTelnetLines.py index 782ef01cfc10b686c0f1b972e2348a5253b653b1..a48351f6c0b162be83f6aca44a87ff9f55b498c8 100644 --- a/tests/pytest/insert/insertTelnetLines.py +++ b/tests/pytest/insert/insertTelnetLines.py @@ -15,7 +15,7 @@ import sys from util.log import * from util.cases import * from util.sql import * - +from util.types import TDSmlProtocolType, TDSmlTimestampType class TDTestCase: def init(self, conn, logSql): @@ -35,10 +35,10 @@ class TDTestCase: "stb0_0 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", "stb0_1 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", "stb0_2 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", - "`.stb0.3.` 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", + ".stb0.3. 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", ] - code = self._conn.schemaless_insert(lines0, 1, None) + code = self._conn.schemaless_insert(lines0, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("show stables") @@ -59,30 +59,26 @@ class TDTestCase: ### timestamp ### print("============= step2 : test timestamp ================") lines1 = [ - "stb1 1626006833s 1i8 host=\"host0\"", - "stb1 1626006833639000000ns 2i8 host=\"host0\"", - "stb1 1626006833640000us 3i8 host=\"host0\"", - "stb1 1626006833641 4i8 host=\"host0\"", - "stb1 1626006834 5i8 host=\"host0\"", - "stb1 1626006833651ms 6i8 host=\"host0\"", - "stb1 0 7i8 host=\"host0\"", + "stb1 1626006833641 1i8 host=\"host0\"", + "stb1 1626006834 2i8 host=\"host0\"", + "stb1 0 3i8 host=\"host0\"", ] - code = self._conn.schemaless_insert(lines1, 1, None) + code = self._conn.schemaless_insert(lines1, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb1") - tdSql.checkRows(7) + tdSql.checkRows(3) ### metric value ### print("============= step3 : test metric value ================") #tinyint lines2_0 = [ - "stb2_0 1626006833651ms -127i8 host=\"host0\"", - "stb2_0 1626006833652ms 127i8 host=\"host0\"" + "stb2_0 1626006833651 -127i8 host=\"host0\"", + "stb2_0 1626006833652 127i8 host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_0, 1, None) + code = self._conn.schemaless_insert(lines2_0, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_0") @@ -94,10 +90,10 @@ class TDTestCase: #smallint lines2_1 = [ - "stb2_1 1626006833651ms -32767i16 host=\"host0\"", - "stb2_1 1626006833652ms 32767i16 host=\"host0\"" + "stb2_1 1626006833651 -32767i16 host=\"host0\"", + "stb2_1 1626006833652 32767i16 host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_1, 1, None) + code = self._conn.schemaless_insert(lines2_1, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_1") @@ -109,11 +105,11 @@ class TDTestCase: #int lines2_2 = [ - "stb2_2 1626006833651ms -2147483647i32 host=\"host0\"", - "stb2_2 1626006833652ms 2147483647i32 host=\"host0\"" + "stb2_2 1626006833651 -2147483647i32 host=\"host0\"", + "stb2_2 1626006833652 2147483647i32 host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_2, 1, None) + code = self._conn.schemaless_insert(lines2_2, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_2") @@ -125,11 +121,11 @@ class TDTestCase: #bigint lines2_3 = [ - "stb2_3 1626006833651ms -9223372036854775807i64 host=\"host0\"", - "stb2_3 1626006833652ms 9223372036854775807i64 host=\"host0\"" + "stb2_3 1626006833651 -9223372036854775807i64 host=\"host0\"", + "stb2_3 1626006833652 9223372036854775807i64 host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_3, 1, None) + code = self._conn.schemaless_insert(lines2_3, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_3") @@ -141,19 +137,19 @@ class TDTestCase: #float lines2_4 = [ - "stb2_4 1626006833610ms 3f32 host=\"host0\"", - "stb2_4 1626006833620ms -3f32 host=\"host0\"", - "stb2_4 1626006833630ms 3.4f32 host=\"host0\"", - "stb2_4 1626006833640ms -3.4f32 host=\"host0\"", - "stb2_4 1626006833650ms 3.4E10f32 host=\"host0\"", - "stb2_4 1626006833660ms -3.4e10f32 host=\"host0\"", - "stb2_4 1626006833670ms 3.4E+2f32 host=\"host0\"", - "stb2_4 1626006833680ms -3.4e-2f32 host=\"host0\"", - "stb2_4 1626006833700ms 3.4E38f32 host=\"host0\"", - "stb2_4 1626006833710ms -3.4E38f32 host=\"host0\"" + "stb2_4 1626006833610 3f32 host=\"host0\"", + "stb2_4 1626006833620 -3f32 host=\"host0\"", + "stb2_4 1626006833630 3.4f32 host=\"host0\"", + "stb2_4 1626006833640 -3.4f32 host=\"host0\"", + "stb2_4 1626006833650 3.4E10f32 host=\"host0\"", + "stb2_4 1626006833660 -3.4e10f32 host=\"host0\"", + "stb2_4 1626006833670 3.4E+2f32 host=\"host0\"", + "stb2_4 1626006833680 -3.4e-2f32 host=\"host0\"", + "stb2_4 1626006833700 3.4E38f32 host=\"host0\"", + "stb2_4 1626006833710 -3.4E38f32 host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_4, 1, None) + code = self._conn.schemaless_insert(lines2_4, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_4") @@ -165,20 +161,20 @@ class TDTestCase: #double lines2_5 = [ - "stb2_5 1626006833610ms 3f64 host=\"host0\"", - "stb2_5 1626006833620ms -3f64 host=\"host0\"", - "stb2_5 1626006833630ms 3.4f64 host=\"host0\"", - "stb2_5 1626006833640ms -3.4f64 host=\"host0\"", - "stb2_5 1626006833650ms 3.4E10f64 host=\"host0\"", - "stb2_5 1626006833660ms -3.4e10f64 host=\"host0\"", - "stb2_5 1626006833670ms 3.4E+2f64 host=\"host0\"", - "stb2_5 1626006833680ms -3.4e-2f64 host=\"host0\"", - "stb2_5 1626006833690ms 1.7E308f64 host=\"host0\"", - "stb2_5 1626006833700ms -1.7E308f64 host=\"host0\"", - "stb2_5 1626006833710ms 3 host=\"host0\"" + "stb2_5 1626006833610 3f64 host=\"host0\"", + "stb2_5 1626006833620 -3f64 host=\"host0\"", + "stb2_5 1626006833630 3.4f64 host=\"host0\"", + "stb2_5 1626006833640 -3.4f64 host=\"host0\"", + "stb2_5 1626006833650 3.4E10f64 host=\"host0\"", + "stb2_5 1626006833660 -3.4e10f64 host=\"host0\"", + "stb2_5 1626006833670 3.4E+2f64 host=\"host0\"", + "stb2_5 1626006833680 -3.4e-2f64 host=\"host0\"", + "stb2_5 1626006833690 1.7E308f64 host=\"host0\"", + "stb2_5 1626006833700 -1.7E308f64 host=\"host0\"", + "stb2_5 1626006833710 3 host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_5, 1, None) + code = self._conn.schemaless_insert(lines2_5, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_5") @@ -190,19 +186,19 @@ class TDTestCase: #bool lines2_6 = [ - "stb2_6 1626006833610ms t host=\"host0\"", - "stb2_6 1626006833620ms T host=\"host0\"", - "stb2_6 1626006833630ms true host=\"host0\"", - "stb2_6 1626006833640ms True host=\"host0\"", - "stb2_6 1626006833650ms TRUE host=\"host0\"", - "stb2_6 1626006833660ms f host=\"host0\"", - "stb2_6 1626006833670ms F host=\"host0\"", - "stb2_6 1626006833680ms false host=\"host0\"", - "stb2_6 1626006833690ms False host=\"host0\"", - "stb2_6 1626006833700ms FALSE host=\"host0\"" + "stb2_6 1626006833610 t host=\"host0\"", + "stb2_6 1626006833620 T host=\"host0\"", + "stb2_6 1626006833630 true host=\"host0\"", + "stb2_6 1626006833640 True host=\"host0\"", + "stb2_6 1626006833650 TRUE host=\"host0\"", + "stb2_6 1626006833660 f host=\"host0\"", + "stb2_6 1626006833670 F host=\"host0\"", + "stb2_6 1626006833680 false host=\"host0\"", + "stb2_6 1626006833690 False host=\"host0\"", + "stb2_6 1626006833700 FALSE host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_6, 1, None) + code = self._conn.schemaless_insert(lines2_6, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_6") @@ -214,12 +210,12 @@ class TDTestCase: #binary lines2_7 = [ - "stb2_7 1626006833610ms \" binary_val .!@#$%^&* \" host=\"host0\"", - "stb2_7 1626006833620ms \"binary_val.:;,./?|+-=\" host=\"host0\"", - "stb2_7 1626006833630ms \"binary_val.()[]{}<>\" host=\"host0\"" + "stb2_7 1626006833610 \" binary_val .!@#$%^&* \" host=\"host0\"", + "stb2_7 1626006833620 \"binary_val.:;,./?|+-=\" host=\"host0\"", + "stb2_7 1626006833630 \"binary_val.()[]{}<>\" host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_7, 1, None) + code = self._conn.schemaless_insert(lines2_7, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_7") @@ -231,11 +227,11 @@ class TDTestCase: #nchar lines2_8 = [ - "stb2_8 1626006833610ms L\" nchar_val 数值一 \" host=\"host0\"", - "stb2_8 1626006833620ms L\"nchar_val数值二\" host=\"host0\"" + "stb2_8 1626006833610 L\" nchar_val 数值一 \" host=\"host0\"", + "stb2_8 1626006833620 L\"nchar_val数值二\" host=\"host0\"" ] - code = self._conn.schemaless_insert(lines2_8, 1, None) + code = self._conn.schemaless_insert(lines2_8, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb2_8") @@ -249,11 +245,11 @@ class TDTestCase: print("============= step3 : test tags ================") #tag value types lines3_0 = [ - "stb3_0 1626006833610ms 1 t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=3.4E38f32 t6=1.7E308f64 t7=true t8=\"binary_val_1\" t9=L\"标签值1\"", - "stb3_0 1626006833610ms 2 t1=-127i8 t2=-32767i16 t3=-2147483647i32 t4=-9223372036854775807i64 t5=-3.4E38f32 t6=-1.7E308f64 t7=false t8=\"binary_val_2\" t9=L\"标签值2\"" + "stb3_0 1626006833610 1 t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=3.4E38f32 t6=1.7E308f64 t7=true t8=\"binary_val_1\" t9=L\"标签值1\"", + "stb3_0 1626006833610 2 t1=-127i8 t2=-32767i16 t3=-2147483647i32 t4=-9223372036854775807i64 t5=-3.4E38f32 t6=-1.7E308f64 t7=false t8=\"binary_val_2\" t9=L\"标签值2\"" ] - code = self._conn.schemaless_insert(lines3_0, 1, None) + code = self._conn.schemaless_insert(lines3_0, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb3_0") @@ -292,12 +288,12 @@ class TDTestCase: #tag ID as child table name lines3_1 = [ - "stb3_1 1626006833610ms 1 id=child_table1 host=host1", - "stb3_1 1626006833610ms 2 host=host2 iD=child_table2", - "stb3_1 1626006833610ms 3 ID=child_table3 host=host3" + "stb3_1 1626006833610 1 id=child_table1 host=host1", + "stb3_1 1626006833610 2 host=host2 iD=child_table2", + "stb3_1 1626006833610 3 ID=child_table3 host=host3" ] - code = self._conn.schemaless_insert(lines3_1, 1, None) + code = self._conn.schemaless_insert(lines3_1, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from stb3_1") @@ -308,6 +304,56 @@ class TDTestCase: tdSql.checkData(0, 0, "child_table1") + ### special characters and keywords ### + print("============= step4 : test special characters and keywords ================") + lines4_1 = [ + "1234 1626006833610ms 1 id=123 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false", + "int 1626006833610ms 2 id=and 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false", + "double 1626006833610ms 2 id=for 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false", + "from 1626006833610ms 2 id=!@#.^& 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false", + "!@#$.%^&*() 1626006833610ms 2 id=none 456=true int=true double=false into=1 from=2 !@#$.%^&*()=false", + "STABLE 1626006833610ms 2 id=KEY 456=true int=true double=false TAG=1 FROM=2 COLUMN=false", + ] + + code = self._conn.schemaless_insert(lines4_1, TDSmlProtocolType.TELNET.value, TDSmlTimestampType.NOT_CONFIGURED.value) + print("schemaless_insert result {}".format(code)) + + tdSql.query('describe `1234`') + tdSql.checkRows(8) + + tdSql.query('describe `int`') + tdSql.checkRows(8) + + tdSql.query('describe `double`') + tdSql.checkRows(8) + + tdSql.query('describe `from`') + tdSql.checkRows(8) + + tdSql.query('describe `!@#$.%^&*()`') + tdSql.checkRows(8) + + tdSql.query('describe `stable`') + tdSql.checkRows(8) + + tdSql.query('select * from `123`') + tdSql.checkRows(1) + + tdSql.query('select * from `and`') + tdSql.checkRows(1) + + tdSql.query('select * from `for`') + tdSql.checkRows(1) + + tdSql.query('select * from `!@#.^&`') + tdSql.checkRows(1) + + tdSql.query('select * from `none`') + tdSql.checkRows(1) + + tdSql.query('select * from `key`') + tdSql.checkRows(1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/insert/line_insert.py b/tests/pytest/insert/line_insert.py index 7a823b917d50c445ddef18ed0f4618f8444d3e85..ff26483aeb323ebd309ba7a41e91ac860af9d222 100644 --- a/tests/pytest/insert/line_insert.py +++ b/tests/pytest/insert/line_insert.py @@ -15,13 +15,14 @@ import sys from util.log import * from util.cases import * from util.sql import * +from util.types import TDSmlProtocolType, TDSmlTimestampType class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self._conn = conn + self._conn = conn def run(self): print("running {}".format(__file__)) @@ -42,17 +43,17 @@ class TDTestCase: "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000" ] - code = self._conn.schemaless_insert(lines, 0, "ns") + code = self._conn.schemaless_insert(lines, TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) print("schemaless_insert result {}".format(code)) lines2 = [ "stg,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000", "stg,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000" ] - code = self._conn.schemaless_insert([ lines2[0] ], 0, "ns") + code = self._conn.schemaless_insert([ lines2[0] ], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) print("schemaless_insert result {}".format(code)) - self._conn.schemaless_insert([ lines2[1] ], 0, "ns") + code = self._conn.schemaless_insert([ lines2[1] ], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) print("schemaless_insert result {}".format(code)) tdSql.query("select * from st") @@ -76,7 +77,7 @@ class TDTestCase: self._conn.schemaless_insert([ "sth,t1=4i64,t2=5f64,t4=5f64,ID=childtable c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641", "sth,t1=4i64,t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933654" - ], 0, "ms") + ], TDSmlProtocolType.LINE.value, TDSmlTimestampType.MILLI_SECOND.value) tdSql.execute('reset query cache') tdSql.query('select tbname, * from sth') @@ -84,6 +85,53 @@ class TDTestCase: tdSql.query('select tbname, * from childtable') tdSql.checkRows(1) + + ###Special Character and keyss + self._conn.schemaless_insert([ + "1234,id=3456,abc=4i64,def=3i64 123=3i64,int=2i64,bool=false,into=5f64,column=7u64,!@#$.%^&*()=false 1626006933641", + "int,id=and,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933654", + "double,id=for,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933664", + "from,id=!@#$.%^,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933674", + "!@#$.%^&*(),id=none,123=4i64,smallint=5f64,double=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false abc=false 1626006933684", + "STABLE,id=CREATE,123=4i64,smallint=5f64,DOUBLE=5f64,of=3i64,key=L\"passitagin_stf\",!@#$.%^&*()=false SELECT=false 1626006933684", + ], TDSmlProtocolType.LINE.value, TDSmlTimestampType.MILLI_SECOND.value) + tdSql.execute('reset query cache') + + tdSql.query('describe `1234`') + tdSql.checkRows(9) + + tdSql.query('describe `int`') + tdSql.checkRows(8) + + tdSql.query('describe `double`') + tdSql.checkRows(8) + + tdSql.query('describe `from`') + tdSql.checkRows(8) + + tdSql.query('describe `!@#$.%^&*()`') + tdSql.checkRows(8) + + tdSql.query('describe `stable`') + tdSql.checkRows(8) + + tdSql.query('select * from `3456`') + tdSql.checkRows(1) + + tdSql.query('select * from `and`') + tdSql.checkRows(1) + + tdSql.query('select * from `for`') + tdSql.checkRows(1) + + tdSql.query('select * from `!@#$.%^`') + tdSql.checkRows(1) + + tdSql.query('select * from `none`') + tdSql.checkRows(1) + + tdSql.query('select * from `create`') + tdSql.checkRows(1) def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 56558ab3be9d74c5abf0987f23b8986a629567b4..94ea0ab79a54cbb7daea1a431fa566567b9de684 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -21,6 +21,7 @@ import numpy as np from util.log import * from util.cases import * from util.sql import * +from util.types import TDSmlProtocolType, TDSmlTimestampType import threading @@ -294,7 +295,7 @@ class TDTestCase: def resCmp(self, input_sql, stb_name, query_sql="select * from", condition="", ts=None, id=True, none_check_tag=None): expect_list = self.inputHandle(input_sql) - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) query_sql = f"{query_sql} {stb_name} {condition}" res_row_list, res_field_list_without_ts, res_type_list = self.resHandle(query_sql, True) if ts == 0: @@ -409,11 +410,11 @@ class TDTestCase: """ for input_sql in [self.genLongSql(128, 1)[0], self.genLongSql(1, 4094)[0]]: self.cleanStb() - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) for input_sql in [self.genLongSql(129, 1)[0], self.genLongSql(1, 4095)[0]]: self.cleanStb() try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -427,7 +428,7 @@ class TDTestCase: for i in rstr: input_sql = self.genFullTypeSql(tb_name=f"\"aaa{i}bbb\"")[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -438,7 +439,7 @@ class TDTestCase: self.cleanStb() input_sql = self.genFullTypeSql(tb_name=f"\"1aaabbb\"")[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -449,7 +450,7 @@ class TDTestCase: self.cleanStb() input_sql = self.genFullTypeSql(ts="now")[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -460,7 +461,7 @@ class TDTestCase: self.cleanStb() input_sql = self.genFullTypeSql(ts="2021-07-21\ 19:01:46.920")[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -471,7 +472,7 @@ class TDTestCase: self.cleanStb() input_sql = self.genFullTypeSql(ts="16260068336390us19")[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -487,7 +488,7 @@ class TDTestCase: for t1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(t1=t1)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -498,7 +499,7 @@ class TDTestCase: for t2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(t2=t2)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -509,7 +510,7 @@ class TDTestCase: for t3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(t3=t3)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -520,7 +521,7 @@ class TDTestCase: for t4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(t4=t4)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError: pass @@ -532,7 +533,7 @@ class TDTestCase: for t5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(t5=t5)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -546,7 +547,7 @@ class TDTestCase: for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -554,11 +555,11 @@ class TDTestCase: # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}" c0=f 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16375, "letters")}" c0=f 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: pass @@ -567,11 +568,11 @@ class TDTestCase: # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}" c0=f 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4094, "letters")}" c0=f 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -589,7 +590,7 @@ class TDTestCase: for c1 in ["-128i8", "128i8"]: input_sql = self.genFullTypeSql(c1=c1)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -600,7 +601,7 @@ class TDTestCase: for c2 in ["-32768i16", "32768i16"]: input_sql = self.genFullTypeSql(c2=c2)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -612,7 +613,7 @@ class TDTestCase: for c3 in ["-2147483648i32", "2147483648i32"]: input_sql = self.genFullTypeSql(c3=c3)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -624,7 +625,7 @@ class TDTestCase: for c4 in ["-9223372036854775808i64", "9223372036854775808i64"]: input_sql = self.genFullTypeSql(c4=c4)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -637,7 +638,7 @@ class TDTestCase: for c5 in [f"{-3.4028234664*(10**38)}f32", f"{3.4028234664*(10**38)}f32"]: input_sql = self.genFullTypeSql(c5=c5)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -650,7 +651,7 @@ class TDTestCase: for c6 in [f'{-1.797693134862316*(10**308)}f64', f'{-1.797693134862316*(10**308)}f64']: input_sql = self.genFullTypeSql(c6=c6)[0] try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -658,11 +659,11 @@ class TDTestCase: # # binary stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}" 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16375, "letters")}" 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -671,11 +672,11 @@ class TDTestCase: # * legal nchar could not be larger than 16374/4 stb_name = self.getLongName(7, "letters") input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}" 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4094, "letters")}" 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -690,13 +691,13 @@ class TDTestCase: for i in ["TrUe", "tRue", "trUe", "truE", "FalsE", "fAlse", "faLse", "falSe", "falsE"]: input_sql1 = self.genFullTypeSql(t0=i)[0] try: - self._conn.schemaless_insert([input_sql1], 0) + self._conn.schemaless_insert([input_sql1], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) input_sql2 = self.genFullTypeSql(c0=i)[0] try: - self._conn.schemaless_insert([input_sql2], 0) + self._conn.schemaless_insert([input_sql2], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -718,7 +719,7 @@ class TDTestCase: self.genFullTypeSql(c9="1s1u64")[0] ]: try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -731,7 +732,7 @@ class TDTestCase: input_sql4 = f'{stb_name},t0=t,t1=L"abc aaa" c0=f 1626006833639000000ns' for input_sql in [input_sql1, input_sql2, input_sql3, input_sql4]: try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -741,8 +742,8 @@ class TDTestCase: for symbol in list('~!@#$¥%^&*()-+={}|[]、「」:;'): input_sql1 = f'{stb_name},t0=t c0=f,c1="abc{symbol}aaa" 1626006833639000000ns' input_sql2 = f'{stb_name},t0=t,t1="abc{symbol}aaa" c0=f 1626006833639000000ns' - self._conn.schemaless_insert([input_sql1], 0) - self._conn.schemaless_insert([input_sql2], 0) + self._conn.schemaless_insert([input_sql1], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + self._conn.schemaless_insert([input_sql2], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) def duplicateIdTagColInsertCheckCase(self): @@ -752,7 +753,7 @@ class TDTestCase: self.cleanStb() input_sql_id = self.genFullTypeSql(id_double_tag=True)[0] try: - self._conn.schemaless_insert([input_sql_id], 0) + self._conn.schemaless_insert([input_sql_id], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -760,7 +761,7 @@ class TDTestCase: input_sql = self.genFullTypeSql()[0] input_sql_tag = input_sql.replace("t5", "t6") try: - self._conn.schemaless_insert([input_sql_tag], 0) + self._conn.schemaless_insert([input_sql_tag], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -768,7 +769,7 @@ class TDTestCase: input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "c6") try: - self._conn.schemaless_insert([input_sql_col], 0) + self._conn.schemaless_insert([input_sql_col], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -776,7 +777,7 @@ class TDTestCase: input_sql = self.genFullTypeSql()[0] input_sql_col = input_sql.replace("c5", "C6") try: - self._conn.schemaless_insert([input_sql_col], 0) + self._conn.schemaless_insert([input_sql_col], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -802,7 +803,7 @@ class TDTestCase: self.cleanStb() input_sql, stb_name = self.genFullTypeSql() self.resCmp(input_sql, stb_name) - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) self.resCmp(input_sql, stb_name) def tagColBinaryNcharLengthCheckCase(self): @@ -869,7 +870,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkEqual(tb_name1, tb_name2) input_sql, stb_name = self.genFullTypeSql(stb_name=stb_name, t0="f", c0="f", id_noexist_tag=True, ct_add_tag=True) - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) tb_name3 = self.getNoIdTbName(stb_name) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) @@ -884,17 +885,17 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) # * every binary and nchar must be length+2, so here is two tag, max length could not larger than 16384-2*2 input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(5, "letters")}" c0=f 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1="{self.getLongName(16374, "letters")}",t2="{self.getLongName(6, "letters")}" c0=f 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError: pass @@ -903,13 +904,13 @@ class TDTestCase: # # * check col，col+ts max in describe ---> 16143 input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(12, "letters")}" 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1="{self.getLongName(16374, "letters")}",c2="{self.getLongName(16374, "letters")}",c3="{self.getLongName(16374, "letters")}",c4="{self.getLongName(13, "letters")}" 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -925,16 +926,16 @@ class TDTestCase: stb_name = self.getLongName(7, "letters") tb_name = f'{stb_name}_1' input_sql = f'{stb_name},id="{tb_name}",t0=t c0=f 1626006833639000000ns' - code = self._conn.schemaless_insert([input_sql], 0) + code = self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) # * legal nchar could not be larger than 16374/4 input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(1, "letters")}" c0=f 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(2) input_sql = f'{stb_name},t0=t,t1=L"{self.getLongName(4093, "letters")}",t2=L"{self.getLongName(2, "letters")}" c0=f 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -942,12 +943,12 @@ class TDTestCase: tdSql.checkRows(2) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(4, "letters")}" 1626006833639000000ns' - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) tdSql.query(f"select * from {stb_name}") tdSql.checkRows(3) input_sql = f'{stb_name},t0=t c0=f,c1=L"{self.getLongName(4093, "letters")}",c2=L"{self.getLongName(4093, "letters")}",c3=L"{self.getLongName(4093, "letters")}",c4=L"{self.getLongName(5, "letters")}" 1626006833639000000ns' try: - self._conn.schemaless_insert([input_sql], 0) + self._conn.schemaless_insert([input_sql], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -971,7 +972,7 @@ class TDTestCase: "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns", "st123456,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933641000000ns" ] - self._conn.schemaless_insert(lines, 0) + self._conn.schemaless_insert(lines, TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) def multiInsertCheckCase(self, count): """ @@ -984,7 +985,7 @@ class TDTestCase: for i in range(count): input_sql = self.genFullTypeSql(stb_name=stb_name, t7=f'"{self.getLongName(8, "letters")}"', c7=f'"{self.getLongName(8, "letters")}"', id_noexist_tag=True)[0] sql_list.append(input_sql) - self._conn.schemaless_insert(sql_list, 0) + self._conn.schemaless_insert(sql_list, TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) def batchErrorInsertCheckCase(self): """ @@ -995,7 +996,7 @@ class TDTestCase: lines = ["st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", f"{stb_name},t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532ns"] try: - self._conn.schemaless_insert(lines, 0) + self._conn.schemaless_insert(lines, TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -1049,7 +1050,7 @@ class TDTestCase: def genMultiThreadSeq(self, sql_list): tlist = list() for insert_sql in sql_list: - t = threading.Thread(target=self._conn.schemaless_insert,args=([insert_sql[0]], 0)) + t = threading.Thread(target=self._conn.schemaless_insert,args=([insert_sql[0]], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value)) tlist.append(t) return tlist @@ -1260,8 +1261,8 @@ class TDTestCase: input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" input_sql2 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64 c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64 1626006933640000000ns" try: - self._conn.schemaless_insert([input_sql1], 0) - self._conn.schemaless_insert([input_sql2], 0) + self._conn.schemaless_insert([input_sql1], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) + self._conn.schemaless_insert([input_sql2], TDSmlProtocolType.LINE.value, TDSmlTimestampType.NANO_SECOND.value) except SchemalessError as err: print(err.errno) # self._conn.schemaless_insert([input_sql2], 0) diff --git a/tests/pytest/query/udf.py b/tests/pytest/query/udf.py index 5b345643b30856195caab938f59c7e8f7a642784..14429a53f44b1393c9f179cc405ed61fb59e8b02 100644 --- a/tests/pytest/query/udf.py +++ b/tests/pytest/query/udf.py @@ -210,10 +210,10 @@ class TDTestCase: tdSql.query("select max(id) + 5 from tb1") tdSql.query("select max(id) + avg(val) from st") tdSql.query("select max(id) + avg(val) from tb1") - tdSql.error("select abs_max(number) + 5 from st") - tdSql.error("select abs_max(number) + 5 from tb1") + tdSql.query("select abs_max(number) + 5 from st") + tdSql.query("select abs_max(number) + 5 from tb1") tdSql.error("select abs_max(number) + max(id) from st") - tdSql.error("select abs_max(number)*abs_max(val) from st") + tdSql.query("select abs_max(number)*abs_max(val) from st") tdLog.info("======= UDF Nested query test =======") tdSql.query("select sum(id) from (select id from st)") diff --git a/tests/pytest/table/create.py b/tests/pytest/table/create.py index 59044bc52b1bb02a33c0a2087deabf5f43d6c518..ec9179c5e97356f284b8d11ed006c12518142328 100644 --- a/tests/pytest/table/create.py +++ b/tests/pytest/table/create.py @@ -298,8 +298,7 @@ class TDTestCase: print("==============step3,#create regular_table; insert regular_table; show regular_table; select regular_table; drop regular_table") self.regular_table = "regular_table~!@#$%^&*()-_+=[]{}';:,<.>/?stST24680~!@#$%^&*()-_+=[]{}" - #self.regular_table = "regular_table~!@#$%^&*()-_+=[]{}';:,<.>/?stST24680~!@#$%^&*()-_+=[]{}" - + tdSql.execute("create table `%s` (ts timestamp,i int) ;" %self.regular_table) tdSql.query("describe `%s` ; "%self.regular_table) tdSql.checkRows(2) @@ -328,9 +327,9 @@ class TDTestCase: tdSql.checkRows(1) self.crr_tb = "create_r_table~!@#$%^&*()-_+=[]{}';:,<.>/?stST24680~!@#$%^&*()-_+=[]{}" - # tdSql.execute("create table `%s` as select * from `%s` ;" %(self.crr_tb,self.regular_table)) - # tdSql.query("show db.tables like 'create_r_table%' ") - # tdSql.checkRows(1) + tdSql.execute("create table `%s` as select * from `%s` ;" %(self.crr_tb,self.regular_table)) + tdSql.query("show db2.tables like 'create_r_table%' ") + tdSql.checkRows(1) print("==============drop table\stable") try: @@ -340,15 +339,6 @@ class TDTestCase: tdSql.error("select * from `%s`" %self.regular_table) - - #表名：192个字符，还要包含前面的数据库名 - #taosdemo 建数据库表 # 单独放 - # self.tsdemo = "tsdemo~!@#$%^&*()-_+=[]{}" - # os.system("%staosdemo -d test -m `%s` -t 10 -n 100 -l 10 -y " % (binPath,self.tsdemo)) - # tdSql.execute("use #!#!#!") - # tdSql.query("select count (tbname) from #!#!#!") - # tdSql.checkData(0, 0, 1000) - diff --git a/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo.py b/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo.py new file mode 100644 index 0000000000000000000000000000000000000000..d7926d6e5b5a3db80f3c66df0655266a5c673999 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo.py @@ -0,0 +1,185 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +import os +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + os.system("rm -rf tools/taosdemoAllTest/TD-10539/create_taosdemo.py.sql") + tdSql.prepare() + + #print("==============taosdemo,#create stable,table; insert table; show table; select table; drop table") + self.tsdemo = "tsdemo~!.@#$%^*[]-_=+{,?.}" + #this escape character is not support in shell . include & () <> | / + os.system("%staosdemo -d test -E -m %s -t 10 -n 100 -l 10 -y " % (binPath,self.tsdemo)) + tdSql.execute("use test ;" ) + tdSql.query("select count(*) from meters") + tdSql.checkData(0, 0, 1000) + tdSql.query("show test.tables like 'tsdemo%'" ) + tdSql.checkRows(10) + tdSql.query("show test.tables like '%s_'" %self.tsdemo) + tdSql.checkRows(10) + tdSql.query("select _block_dist() from `%s1`" %self.tsdemo) + tdSql.checkRows(1) + tdSql.query("describe test.`%s1` ; " %self.tsdemo) + tdSql.checkRows(13) + tdSql.query("show create table test.`%s1` ; " %self.tsdemo) + tdSql.checkData(0, 0, self.tsdemo+str(1)) + tdSql.checkData(0, 1, "CREATE TABLE `%s1` USING `meters` TAGS (1,\"beijing\")" %self.tsdemo) + + print("==============drop table\stable") + try: + tdSql.execute("drop table test.`%s1` ; " %self.tsdemo) + except Exception as e: + tdLog.exit(e) + + tdSql.error("select * from test.`%s1` ; " %self.tsdemo) + tdSql.query("show test.tables like '%s_'" %self.tsdemo) + tdSql.checkRows(9) + + try: + tdSql.execute("drop table test.meters ") + except Exception as e: + tdLog.exit(e) + + tdSql.error("select * from test.meters ") + tdSql.error("select * from test.`%s2` ; " %self.tsdemo) + + # Exception + os.system("%staosdemo -d test -m %s -t 10 -n 100 -l 10 -y " % (binPath,self.tsdemo)) + tdSql.query("show test.tables ") + tdSql.checkRows(0) + + #print("==============taosdemo,#create regular table; insert table; show table; select table; drop table") + self.tsdemo = "tsdemo~!.@#$%^*[]-_=+{,?.}" + #this escape character is not support in shell . include & () <> | / + os.system("%staosdemo -N -E -m %s -t 10 -n 100 -l 10 -y " % (binPath,self.tsdemo)) + tdSql.execute("use test ;" ) + tdSql.query("select count(*) from `%s1`" %self.tsdemo) + tdSql.checkData(0, 0, 100) + tdSql.query("show test.tables like 'tsdemo%'" ) + tdSql.checkRows(10) + tdSql.query("show test.tables like '%s_'" %self.tsdemo) + tdSql.checkRows(10) + tdSql.query("select _block_dist() from `%s1`" %self.tsdemo) + tdSql.checkRows(1) + tdSql.query("describe test.`%s1` ; " %self.tsdemo) + tdSql.checkRows(11) + tdSql.query("show create table test.`%s1` ; " %self.tsdemo) + tdSql.checkData(0, 0, self.tsdemo+str(1)) + tdSql.checkData(0, 1, "create table `%s1` (ts TIMESTAMP,c0 FLOAT,c1 INT,c2 INT,c3 INT,c4 INT,c5 INT,c6 INT,c7 INT,c8 INT,c9 INT)" %self.tsdemo) + + print("==============drop table\stable") + try: + tdSql.execute("drop table test.`%s1` ; " %self.tsdemo) + except Exception as e: + tdLog.exit(e) + + tdSql.error("select * from test.`%s1` ; " %self.tsdemo) + tdSql.query("show test.tables like '%s_'" %self.tsdemo) + tdSql.checkRows(9) + + # Exception + os.system("%staosdemo -N -m %s -t 10 -n 100 -l 10 -y " % (binPath,self.tsdemo)) + tdSql.query("show test.tables ") + tdSql.checkRows(0) + + + #print("==============taosdemo——json_yes,#create stable,table; insert table; show table; select table; drop table") + os.system("%staosdemo -f tools/taosdemoAllTest/TD-10539/create_taosdemo_yes.json -y " % binPath) + tdSql.execute("use dbyes") + + self.tsdemo_stable = "tsdemo_stable~!.@#$%^*[]-_=+{,?.}" + self.tsdemo = "tsdemo~!.@#$%^*[]-_=+{,?.}" + + tdSql.query("select count(*) from dbyes.`%s`" %self.tsdemo_stable) + tdSql.checkData(0, 0, 1000) + tdSql.query("show dbyes.tables like 'tsdemo%'" ) + tdSql.checkRows(10) + tdSql.query("show dbyes.tables like '%s_'" %self.tsdemo) + tdSql.checkRows(10) + tdSql.query("select _block_dist() from `%s1`" %self.tsdemo) + tdSql.checkRows(1) + tdSql.query("describe dbyes.`%s1` ; " %self.tsdemo) + tdSql.checkRows(13) + tdSql.query("show create table dbyes.`%s1` ; " %self.tsdemo) + tdSql.checkData(0, 0, self.tsdemo+str(1)) + tdSql.checkData(0, 1, "CREATE TABLE `%s1` USING `%s` TAGS (1,1)" %(self.tsdemo,self.tsdemo_stable)) + + print("==============drop table\stable") + try: + tdSql.execute("drop table dbyes.`%s1` ; " %self.tsdemo) + except Exception as e: + tdLog.exit(e) + + tdSql.error("select * from dbyes.`%s1` ; " %self.tsdemo) + tdSql.query("show dbyes.tables like '%s_'" %self.tsdemo) + tdSql.checkRows(9) + + try: + tdSql.execute("drop table dbyes.`%s` ; " %self.tsdemo_stable) + except Exception as e: + tdLog.exit(e) + + tdSql.error("select * from dbyes.`%s` ; " %self.tsdemo_stable) + tdSql.error("select * from dbyes.`%s2` ; " %self.tsdemo) + + #print("==============taosdemo——json_no,#create stable,table; insert table; show table; select table; drop table") + + assert os.system("%staosdemo -f tools/taosdemoAllTest/TD-10539/create_taosdemo_no.json -y " % binPath) == 0 + tdSql.query("show dbno.tables ") + tdSql.checkRows(0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo_no.json b/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo_no.json new file mode 100644 index 0000000000000000000000000000000000000000..759a437b448c8c65bf252e859345dd9557cc51c5 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo_no.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "dbno", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 36500, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "meters", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "tsdemo~!.@#$%^*[]-_=+{,?.}", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 1, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT","count":9}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "INT", "count":2}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo_yes.json b/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo_yes.json new file mode 100644 index 0000000000000000000000000000000000000000..aafc79215fc0b94d037da3a9b229a2f967b51613 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-10539/create_taosdemo_yes.json @@ -0,0 +1,63 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 5, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "dbyes", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 36500, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "tsdemo_stable~!.@#$%^*[]-_=+{,?.}", + "child_table_exists":"no", + "childtable_count": 10, + "childtable_prefix": "tsdemo~!.@#$%^*[]-_=+{,?.}", + "escape_character": "yes", + "auto_create_table": "no", + "batch_create_tbl_num": 1, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "", + "tags_file": "", + "columns": [{"type": "INT","count":9}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "INT", "count":2}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json index 5e53bd7e7d10edea9bdbc56ef9ab737dbb34684e..c2e4920097cd1b3581c9893c9677c3cf1f14b7ed 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "ms", - "keep": 365, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, @@ -36,7 +36,7 @@ "name": "stb0", "child_table_exists":"no", "childtable_count": 60, - "childtable_prefix": "stb00_", + "childtable_prefix": "stb00_", "auto_create_table": "no", "batch_create_tbl_num": 20, "data_source": "rand", diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json index 49ab6f3a4367b4cebd840bb24b43a5d190c0d464..fd458a88d1a434c22958d5086949009cdd6080bf 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "ms", - "keep": 36, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json index 9a35df917dcbb2600852e8172da0be3ffacb0d15..99233bdd738d068664241efda40d96c5a6fc7090 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "ns", - "keep": 36, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json index 631179dbaebfff29de6b38831b78fede989369d4..14bb9e9be07d9bd61dc089af0bb34acd523155d9 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "us", - "keep": 36, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json index 246f1c35f29973fc20602284b37ae68de23f70c1..e6c4b3205a77e20714067733bfa6f6c4053f087c 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "ns", - "keep": 36, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json index 0726f3905de2b254b49be51a7973d34b5eb6757e..a19132b1da9c99b8fe3792a1c2d475fd4f18ef91 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "ns", - "keep": 36, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json index f36b1f9b4c1b83707b9482428d4303a5418ad2c3..3b4c43d5d05ee1a1b26ee4016b1c38aade592b56 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "ns", - "keep": 36, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json index 867619ed8c1497e76077f96d257dd09a489d9eb7..7fb90727ef6fa38da73639ebe11125924b9ed507 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json @@ -22,7 +22,7 @@ "cache": 50, "blocks": 8, "precision": "ns", - "keep": 36, + "keep": 36500, "minRows": 100, "maxRows": 4096, "comp":2, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py index 6021c9136ad235f3e9d07bb4f6654fdac54989e5..3a3152ecde3c4eca09d8b8583cf90bbfdc0cc31d 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestQueryWithJson.py @@ -20,14 +20,16 @@ from util.dnodes import * import time from datetime import datetime import ast +import re # from assertpy import assert_that import subprocess + class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -40,52 +42,54 @@ class TDTestCase: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + buildPath = root[:len(root) - len("/build/bin")] break return buildPath # 获取taosc接口查询的结果文件中的内容,返回每行数据,并断言数据的第一列内容。 - def assertfileDataTaosc(self,filename,expectResult): + def assertfileDataTaosc(self, filename, expectResult): self.filename = filename self.expectResult = expectResult - with open("%s" % filename, 'r+') as f1: + with open("%s" % filename, 'r+') as f1: for line in f1.readlines(): queryResult = line.strip().split()[0] - self.assertCheck(filename,queryResult,expectResult) + self.assertCheck(filename, queryResult, expectResult) # 获取restful接口查询的结果文件中的关键内容,目前的关键内容找到第一个key就跳出循，所以就只有一个数据。后续再修改多个结果文件。 - def getfileDataRestful(self,filename): + def getfileDataRestful(self, filename): self.filename = filename - with open("%s" % filename, 'r+') as f1: + with open("%s" % filename, 'r+') as f1: for line in f1.readlines(): contents = line.strip() if contents.find("data") != -1: + pattern = re.compile("{.*}") + contents = pattern.search(contents).group() contentsDict = ast.literal_eval(contents) # 字符串转换为字典 queryResult = contentsDict['data'][0][0] break return queryResult # 获取taosc接口查询次数 - def queryTimesTaosc(self,filename): + def queryTimesTaosc(self, filename): self.filename = filename - command = 'cat %s |wc -l'% filename - times = int(subprocess.getstatusoutput(command)[1]) + command = 'cat %s |wc -l' % filename + times = int(subprocess.getstatusoutput(command)[1]) return times # 获取restful接口查询次数 - def queryTimesRestful(self,filename): + def queryTimesRestful(self, filename): self.filename = filename - command = 'cat %s |grep "200 OK" |wc -l'% filename - times = int(subprocess.getstatusoutput(command)[1]) + command = 'cat %s |grep "200 OK" |wc -l' % filename + times = int(subprocess.getstatusoutput(command)[1]) return times # 定义断言结果是否正确。不正确返回错误结果，正确即通过。 - def assertCheck(self,filename,queryResult,expectResult): + def assertCheck(self, filename, queryResult, expectResult): self.filename = filename self.queryResult = queryResult self.expectResult = expectResult args0 = (filename, queryResult, expectResult) - assert queryResult == expectResult , "Queryfile:%s ,result is %s != expect: %s" % args0 + assert queryResult == expectResult, "Queryfile:%s ,result is %s != expect: %s" % args0 def run(self): buildPath = self.getBuildPath() @@ -93,109 +97,144 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - + binPath = buildPath + "/build/bin/" + # delete useless files - os.system("rm -rf ./query_res*") + os.system("rm -rf ./query_res*") os.system("rm -rf ./all_query*") - - # taosc query: query specified table and query super table - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/queryTaosc.json" % binPath) + + # taosc query: query specified table and query super table + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryTaosc.json" % + binPath) os.system("cat query_res0.txt* > all_query_res0_taosc.txt") os.system("cat query_res1.txt* > all_query_res1_taosc.txt") os.system("cat query_res2.txt* > all_query_res2_taosc.txt") - - # correct Times testcases + + # correct Times testcases queryTimes0Taosc = self.queryTimesTaosc("all_query_res0_taosc.txt") - self.assertCheck("all_query_res0_taosc.txt",queryTimes0Taosc,6) + self.assertCheck("all_query_res0_taosc.txt", queryTimes0Taosc, 6) queryTimes1Taosc = self.queryTimesTaosc("all_query_res1_taosc.txt") - self.assertCheck("all_query_res1_taosc.txt",queryTimes1Taosc,6) + self.assertCheck("all_query_res1_taosc.txt", queryTimes1Taosc, 6) queryTimes2Taosc = self.queryTimesTaosc("all_query_res2_taosc.txt") - self.assertCheck("all_query_res2_taosc.txt",queryTimes2Taosc,20) - + self.assertCheck("all_query_res2_taosc.txt", queryTimes2Taosc, 20) + # correct data testcase - self.assertfileDataTaosc("all_query_res0_taosc.txt","1604160000099") - self.assertfileDataTaosc("all_query_res1_taosc.txt","100") - self.assertfileDataTaosc("all_query_res2_taosc.txt","1604160000199") - + self.assertfileDataTaosc("all_query_res0_taosc.txt", "1604160000099") + self.assertfileDataTaosc("all_query_res1_taosc.txt", "100") + self.assertfileDataTaosc("all_query_res2_taosc.txt", "1604160000199") + # delete useless files - os.system("rm -rf ./query_res*") + os.system("rm -rf ./query_res*") os.system("rm -rf ./all_query*") - # use restful api to query - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertrestdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/queryRestful.json" % binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertrestdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryRestful.json" % + binPath) os.system("cat query_res0.txt* > all_query_res0_rest.txt") os.system("cat query_res1.txt* > all_query_res1_rest.txt") os.system("cat query_res2.txt* > all_query_res2_rest.txt") - - # correct Times testcases + + # correct Times testcases queryTimes0Restful = self.queryTimesRestful("all_query_res0_rest.txt") - self.assertCheck("all_query_res0_rest.txt",queryTimes0Restful,6) + self.assertCheck("all_query_res0_rest.txt", queryTimes0Restful, 6) queryTimes1Restful = self.queryTimesRestful("all_query_res1_rest.txt") - self.assertCheck("all_query_res1_rest.txt",queryTimes1Restful,6) - + self.assertCheck("all_query_res1_rest.txt", queryTimes1Restful, 6) + queryTimes2Restful = self.queryTimesRestful("all_query_res2_rest.txt") - self.assertCheck("all_query_res2_rest.txt",queryTimes2Restful,4) + self.assertCheck("all_query_res2_rest.txt", queryTimes2Restful, 4) # correct data testcase data0 = self.getfileDataRestful("all_query_res0_rest.txt") - self.assertCheck('all_query_res0_rest.txt',data0,"2020-11-01 00:00:00.009") + self.assertCheck( + 'all_query_res0_rest.txt', + data0, + "2020-11-01 00:00:00.009") data1 = self.getfileDataRestful("all_query_res1_rest.txt") - self.assertCheck('all_query_res1_rest.txt',data1,10) + self.assertCheck('all_query_res1_rest.txt', data1, 10) data2 = self.getfileDataRestful("all_query_res2_rest.txt") - self.assertCheck('all_query_res2_rest.txt',data2,"2020-11-01 00:00:00.004") - - + self.assertCheck( + 'all_query_res2_rest.txt', + data2, + "2020-11-01 00:00:00.004") + # query times less than or equal to 100 - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/querySpeciMutisql100.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/querySuperMutisql100.json" % binPath) - - #query result print QPS - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - os.system("%staosdemo -f tools/taosdemoAllTest/queryQps.json" % binPath) - + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/querySpeciMutisql100.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/querySuperMutisql100.json" % + binPath) + + # query result print QPS + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryQps.json" % + binPath) + # use illegal or out of range parameters query json file - os.system("%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % binPath) - exceptcode = os.system("%staosdemo -f tools/taosdemoAllTest/queryTimes0.json" % binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/queryInsertdata.json" % + binPath) + exceptcode = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryTimes0.json" % + binPath) assert exceptcode != 0 - exceptcode0 = os.system("%staosdemo -f tools/taosdemoAllTest/queryTimesless0.json" % binPath) + exceptcode0 = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryTimesless0.json" % + binPath) assert exceptcode0 != 0 - exceptcode1 = os.system("%staosdemo -f tools/taosdemoAllTest/queryConcurrentless0.json" % binPath) + exceptcode1 = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryConcurrentless0.json" % + binPath) assert exceptcode1 != 0 - exceptcode2 = os.system("%staosdemo -f tools/taosdemoAllTest/queryConcurrent0.json" % binPath) + exceptcode2 = os.system( + "%staosdemo -f tools/taosdemoAllTest/queryConcurrent0.json" % + binPath) assert exceptcode2 != 0 - exceptcode3 = os.system("%staosdemo -f tools/taosdemoAllTest/querrThreadsless0.json" % binPath) + exceptcode3 = os.system( + "%staosdemo -f tools/taosdemoAllTest/querrThreadsless0.json" % + binPath) assert exceptcode3 != 0 - exceptcode4 = os.system("%staosdemo -f tools/taosdemoAllTest/querrThreads0.json" % binPath) + exceptcode4 = os.system( + "%staosdemo -f tools/taosdemoAllTest/querrThreads0.json" % + binPath) assert exceptcode4 != 0 # delete useless files os.system("rm -rf ./insert_res.txt") - os.system("rm -rf tools/taosdemoAllTest/*.py.sql") - os.system("rm -rf ./querySystemInfo*") - os.system("rm -rf ./query_res*") - os.system("rm -rf ./all_query*") + os.system("rm -rf tools/taosdemoAllTest/*.py.sql") + os.system("rm -rf ./querySystemInfo*") + os.system("rm -rf ./query_res*") +# os.system("rm -rf ./all_query*") os.system("rm -rf ./test_query_res0.txt") - - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) + tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py index c3fdff00ec15fc1ca0d55f86d430c5cbf86ad168..d8c68af0f9b43443744d7d799db6f5ee1e1dacaa 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanoInsert.py @@ -36,7 +36,7 @@ class TDTestCase: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + buildPath = root[:len(root) - len("/build/bin")] break return buildPath @@ -46,14 +46,15 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" + binPath = buildPath + "/build/bin/" - # insert: create one or mutiple tables per sql and insert multiple rows per sql # insert data from a special timestamp # check stable stb0 - os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabase.json -y " % binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabase.json -y " % + binPath) tdSql.execute("use nsdb") tdSql.query("show stables") tdSql.checkData(0, 4, 100) @@ -64,9 +65,9 @@ class TDTestCase: tdSql.query("select count(*) from stb0") tdSql.checkData(0, 0, 10000) tdSql.query("describe stb0") - tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.checkDataType(9, 1, "TIMESTAMP") tdSql.query("select last(ts) from stb0") - tdSql.checkData(0, 0,"2021-07-01 00:00:00.990000000") + tdSql.checkData(0, 0, "2021-07-01 00:00:00.990000000") # check stable stb1 which is insert with disord @@ -78,16 +79,18 @@ class TDTestCase: tdSql.checkData(0, 0, 10000) # check c8 is an nano timestamp tdSql.query("describe stb1") - tdSql.checkDataType(9, 1,"TIMESTAMP") + tdSql.checkDataType(9, 1, "TIMESTAMP") # check insert timestamp_step is nano_second tdSql.query("select last(ts) from stb1") - tdSql.checkData(0, 0,"2021-07-01 00:00:00.990000000") - + tdSql.checkData(0, 0, "2021-07-01 00:00:00.990000000") + # insert data from now time # check stable stb0 - os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json -y " % binPath) - + os.system( + "%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json -y " % + binPath) + tdSql.execute("use nsdb2") tdSql.query("show stables") tdSql.checkData(0, 4, 100) @@ -99,11 +102,14 @@ class TDTestCase: tdSql.checkData(0, 0, 10000) # check c8 is an nano timestamp tdSql.query("describe stb0") - tdSql.checkDataType(9,1,"TIMESTAMP") + tdSql.checkDataType(9, 1, "TIMESTAMP") + + # insert by csv files and timetamp is long int , strings in ts and + # cols - # insert by csv files and timetamp is long int , strings in ts and cols - - os.system("%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json -y " % binPath) + os.system( + "%staosdemo -f tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json -y " % + binPath) tdSql.execute("use nsdbcsv") tdSql.query("show stables") tdSql.checkData(0, 4, 100) @@ -111,29 +117,36 @@ class TDTestCase: tdSql.checkData(0, 0, 10000) tdSql.query("describe stb0") tdSql.checkDataType(3, 1, "TIMESTAMP") - tdSql.query("select count(*) from stb0 where ts > \"2021-07-01 00:00:00.490000000\"") + tdSql.query( + "select count(*) from stb0 where ts > \"2021-07-01 00:00:00.490000000\"") tdSql.checkData(0, 0, 5000) tdSql.query("select count(*) from stb0 where ts < 1626918583000000000") tdSql.checkData(0, 0, 10000) - + os.system("rm -rf ./insert_res.txt") os.system("rm -rf tools/taosdemoAllTest/taosdemoTestSupportNano*.py.sql") - # taosdemo test insert with command and parameter , detals show taosdemo --help - os.system("%staosdemo -u root -ptaosdata -P 6030 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % binPath) + # taosdemo test insert with command and parameter , detals show + # taosdemo --help + os.system( + "%staosdemo -u root -ptaosdata -P 6030 -a 1 -m pre -n 10 -T 20 -t 60 -o res.txt -y " % + binPath) tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, 600) # check taosdemo -s - sqls_ls = ['drop database if exists nsdbsql;','create database nsdbsql precision "ns" keep 36 days 6 update 1;', - 'use nsdbsql;','CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);', - 'CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);', - 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);', - 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 85, 32, 0.76);'] + sqls_ls = [ + 'drop database if exists nsdbsql;', + 'create database nsdbsql precision "ns" keep 36500 days 6 update 1;', + 'use nsdbsql;', + 'CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);', + 'CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2);', + 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32);', + 'INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 85, 32, 0.76);'] - with open("./taosdemoTestNanoCreateDB.sql",mode ="a" ) as sql_files: + with open("./taosdemoTestNanoCreateDB.sql", mode="a") as sql_files: for sql in sqls_ls: - sql_files.write(sql+"\n") + sql_files.write(sql + "\n") sql_files.close() sleep(10) @@ -141,11 +154,10 @@ class TDTestCase: os.system("%staosdemo -s taosdemoTestNanoCreateDB.sql -y " % binPath) tdSql.query("select count(*) from nsdbsql.meters") tdSql.checkData(0, 0, 2) - + os.system("rm -rf ./res.txt") os.system("rm -rf ./*.py.sql") os.system("rm -rf ./taosdemoTestNanoCreateDB.sql") - def stop(self): tdSql.close() diff --git a/tests/pytest/util/types.py b/tests/pytest/util/types.py new file mode 100644 index 0000000000000000000000000000000000000000..218a4770269328a5ef7161cc56c0e0dc0c420f73 --- /dev/null +++ b/tests/pytest/util/types.py @@ -0,0 +1,38 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from enum import Enum + +class TDSmlProtocolType(Enum): + ''' + Schemaless Protocol types + 0 - unknown + 1 - InfluxDB Line Protocol + 2 - OpenTSDB Telnet Protocl + 3 - OpenTSDB JSON Protocol + ''' + UNKNOWN = 0 + LINE = 1 + TELNET = 2 + JSON = 3 + +class TDSmlTimestampType(Enum): + NOT_CONFIGURED = 0 + HOUR = 1 + MINUTE = 2 + SECOND = 3 + MILLI_SECOND = 4 + MICRO_SECOND = 5 + NANO_SECOND = 6 + + diff --git a/tests/script/api/openTSDBTest.c b/tests/script/api/openTSDBTest.c index 2b9cf986f2f5278f1cfc1c8042d735423fdef312..8b70a324ea55c905c9a8bdaf67de9c258f9d57d7 100644 --- a/tests/script/api/openTSDBTest.c +++ b/tests/script/api/openTSDBTest.c @@ -22,168 +22,190 @@ void verify_telnet_insert(TAOS* taos) { /* metric */ char* lines0[] = { - "stb0_0 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", - "stb0_1 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", - "stb0_2 1626006833639000000ns 4i8 host=\"host0\" interface=\"eth0\"", + "stb0_0 1626006833639 4i8 host=\"host0\" interface=\"eth0\"", + "stb0_1 1626006833639 4i8 host=\"host0\" interface=\"eth0\"", + "stb0_2 1626006833639 4i8 host=\"host0\" interface=\"eth0\"", }; - code = taos_schemaless_insert(taos, lines0, 3, 1, NULL); + result = taos_schemaless_insert(taos, lines0, 3, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines0 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); /* timestamp */ char* lines1[] = { - "stb1 1626006833s 1i8 host=\"host0\"", - "stb1 1626006833639000000ns 2i8 host=\"host0\"", - "stb1 1626006833640000us 3i8 host=\"host0\"", - "stb1 1626006833641 4i8 host=\"host0\"", - "stb1 1626006832 5i8 host=\"host0\"", - "stb1 1626006833651ms 6i8 host=\"host0\"", - "stb1 0 7i8 host=\"host0\"", + "stb1 1626006833641 1i8 host=\"host0\"", + "stb1 1626006832 2i8 host=\"host0\"", + "stb1 0 3i8 host=\"host0\"", }; - code = taos_schemaless_insert(taos, lines1, 7, 1, NULL); + result = taos_schemaless_insert(taos, lines1, 3, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines1 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); /* metric value */ //tinyint char* lines2_0[] = { - "stb2_0 1626006833651ms -127i8 host=\"host0\"", - "stb2_0 1626006833652ms 127i8 host=\"host0\"" + "stb2_0 1626006833651 -127i8 host=\"host0\"", + "stb2_0 1626006833652 127i8 host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_0, 2, 1, NULL); + result = taos_schemaless_insert(taos, lines2_0, 2, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_0 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //smallint char* lines2_1[] = { - "stb2_1 1626006833651ms -32767i16 host=\"host0\"", - "stb2_1 1626006833652ms 32767i16 host=\"host0\"" + "stb2_1 1626006833651 -32767i16 host=\"host0\"", + "stb2_1 1626006833652 32767i16 host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_1, 2, 1, NULL); + result = taos_schemaless_insert(taos, lines2_1, 2, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_1 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //int char* lines2_2[] = { - "stb2_2 1626006833651ms -2147483647i32 host=\"host0\"", - "stb2_2 1626006833652ms 2147483647i32 host=\"host0\"" + "stb2_2 1626006833651 -2147483647i32 host=\"host0\"", + "stb2_2 1626006833652 2147483647i32 host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_2, 2, 1, NULL); + result = taos_schemaless_insert(taos, lines2_2, 2, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_2 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //bigint char* lines2_3[] = { - "stb2_3 1626006833651ms -9223372036854775807i64 host=\"host0\"", - "stb2_3 1626006833652ms 9223372036854775807i64 host=\"host0\"" + "stb2_3 1626006833651 -9223372036854775807i64 host=\"host0\"", + "stb2_3 1626006833652 9223372036854775807i64 host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_3, 2, 1, NULL); + result = taos_schemaless_insert(taos, lines2_3, 2, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_3 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //float char* lines2_4[] = { - "stb2_4 1626006833610ms 3f32 host=\"host0\"", - "stb2_4 1626006833620ms -3f32 host=\"host0\"", - "stb2_4 1626006833630ms 3.4f32 host=\"host0\"", - "stb2_4 1626006833640ms -3.4f32 host=\"host0\"", - "stb2_4 1626006833650ms 3.4E10f32 host=\"host0\"", - "stb2_4 1626006833660ms -3.4e10f32 host=\"host0\"", - "stb2_4 1626006833670ms 3.4E+2f32 host=\"host0\"", - "stb2_4 1626006833680ms -3.4e-2f32 host=\"host0\"", - "stb2_4 1626006833700ms 3.4E38f32 host=\"host0\"", - "stb2_4 1626006833710ms -3.4E38f32 host=\"host0\"" + "stb2_4 1626006833610 3f32 host=\"host0\"", + "stb2_4 1626006833620 -3f32 host=\"host0\"", + "stb2_4 1626006833630 3.4f32 host=\"host0\"", + "stb2_4 1626006833640 -3.4f32 host=\"host0\"", + "stb2_4 1626006833650 3.4E10f32 host=\"host0\"", + "stb2_4 1626006833660 -3.4e10f32 host=\"host0\"", + "stb2_4 1626006833670 3.4E+2f32 host=\"host0\"", + "stb2_4 1626006833680 -3.4e-2f32 host=\"host0\"", + "stb2_4 1626006833700 3.4E38f32 host=\"host0\"", + "stb2_4 1626006833710 -3.4E38f32 host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_4, 10, 1, NULL); + result = taos_schemaless_insert(taos, lines2_4, 10, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_4 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //double char* lines2_5[] = { - "stb2_5 1626006833610ms 3f64 host=\"host0\"", - "stb2_5 1626006833620ms -3f64 host=\"host0\"", - "stb2_5 1626006833630ms 3.4f64 host=\"host0\"", - "stb2_5 1626006833640ms -3.4f64 host=\"host0\"", - "stb2_5 1626006833650ms 3.4E10f64 host=\"host0\"", - "stb2_5 1626006833660ms -3.4e10f64 host=\"host0\"", - "stb2_5 1626006833670ms 3.4E+2f64 host=\"host0\"", - "stb2_5 1626006833680ms -3.4e-2f64 host=\"host0\"", - "stb2_5 1626006833690ms 1.7E308f64 host=\"host0\"", - "stb2_5 1626006833700ms -1.7E308f64 host=\"host0\"", - "stb2_5 1626006833710ms 3.15 host=\"host0\"" + "stb2_5 1626006833610 3f64 host=\"host0\"", + "stb2_5 1626006833620 -3f64 host=\"host0\"", + "stb2_5 1626006833630 3.4f64 host=\"host0\"", + "stb2_5 1626006833640 -3.4f64 host=\"host0\"", + "stb2_5 1626006833650 3.4E10f64 host=\"host0\"", + "stb2_5 1626006833660 -3.4e10f64 host=\"host0\"", + "stb2_5 1626006833670 3.4E+2f64 host=\"host0\"", + "stb2_5 1626006833680 -3.4e-2f64 host=\"host0\"", + "stb2_5 1626006833690 1.7E308f64 host=\"host0\"", + "stb2_5 1626006833700 -1.7E308f64 host=\"host0\"", + "stb2_5 1626006833710 3.15 host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_5, 11, 1, NULL); + result = taos_schemaless_insert(taos, lines2_5, 11, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_5 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //bool char* lines2_6[] = { - "stb2_6 1626006833610ms t host=\"host0\"", - "stb2_6 1626006833620ms T host=\"host0\"", - "stb2_6 1626006833630ms true host=\"host0\"", - "stb2_6 1626006833640ms True host=\"host0\"", - "stb2_6 1626006833650ms TRUE host=\"host0\"", - "stb2_6 1626006833660ms f host=\"host0\"", - "stb2_6 1626006833670ms F host=\"host0\"", - "stb2_6 1626006833680ms false host=\"host0\"", - "stb2_6 1626006833690ms False host=\"host0\"", - "stb2_6 1626006833700ms FALSE host=\"host0\"" + "stb2_6 1626006833610 t host=\"host0\"", + "stb2_6 1626006833620 T host=\"host0\"", + "stb2_6 1626006833630 true host=\"host0\"", + "stb2_6 1626006833640 True host=\"host0\"", + "stb2_6 1626006833650 TRUE host=\"host0\"", + "stb2_6 1626006833660 f host=\"host0\"", + "stb2_6 1626006833670 F host=\"host0\"", + "stb2_6 1626006833680 false host=\"host0\"", + "stb2_6 1626006833690 False host=\"host0\"", + "stb2_6 1626006833700 FALSE host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_6, 10, 1, NULL); + result = taos_schemaless_insert(taos, lines2_6, 10, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_6 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //binary char* lines2_7[] = { - "stb2_7 1626006833610ms \"binary_val.!@#$%^&*\" host=\"host0\"", - "stb2_7 1626006833620ms \"binary_val.:;,./?|+-=\" host=\"host0\"", - "stb2_7 1626006833630ms \"binary_val.()[]{}<>\" host=\"host0\"" + "stb2_7 1626006833610 \"binary_val.!@#$%^&*\" host=\"host0\"", + "stb2_7 1626006833620 \"binary_val.:;,./?|+-=\" host=\"host0\"", + "stb2_7 1626006833630 \"binary_val.()[]{}<>\" host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_7, 3, 1, NULL); + result = taos_schemaless_insert(taos, lines2_7, 3, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_7 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //nchar char* lines2_8[] = { - "stb2_8 1626006833610ms L\"nchar_val数值一\" host=\"host0\"", - "stb2_8 1626006833620ms L\"nchar_val数值二\" host=\"host0\"" + "stb2_8 1626006833610 L\"nchar_val数值一\" host=\"host0\"", + "stb2_8 1626006833620 L\"nchar_val数值二\" host=\"host0\"" }; - code = taos_schemaless_insert(taos, lines2_8, 2, 1, NULL); + result = taos_schemaless_insert(taos, lines2_8, 2, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines2_8 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); /* tags */ //tag value types char* lines3_0[] = { - "stb3_0 1626006833610ms 1 t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=3.4E38f32 t6=1.7E308f64 t7=true t8=\"binary_val_1\" t9=L\"标签值1\"", - "stb3_0 1626006833610ms 2 t1=-127i8 t2=-32767i16 t3=-2147483647i32 t4=-9223372036854775807i64 t5=-3.4E38f32 t6=-1.7E308f64 t7=false t8=\"binary_val_2\" t9=L\"标签值2\"" + "stb3_0 1626006833610 1 t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=3.4E38f32 t6=1.7E308f64 t7=true t8=\"binary_val_1\" t9=L\"标签值1\"", + "stb3_0 1626006833610 2 t1=-127i8 t2=-32767i16 t3=-2147483647i32 t4=-9223372036854775807i64 t5=-3.4E38f32 t6=-1.7E308f64 t7=false t8=\"binary_val_2\" t9=L\"标签值2\"" }; - code = taos_schemaless_insert(taos, lines3_0, 2, 1, NULL); + result = taos_schemaless_insert(taos, lines3_0, 2, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines3_0 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); //tag ID as child table name char* lines3_1[] = { - "stb3_1 1626006833610ms 1 id=child_table1 host=host1", - "stb3_1 1626006833610ms 2 host=host2 iD=child_table2", - "stb3_1 1626006833610ms 3 ID=child_table3 host=host3" + "stb3_1 1626006833610 1 id=child_table1 host=host1", + "stb3_1 1626006833610 2 host=host2 iD=child_table2", + "stb3_1 1626006833610 3 ID=child_table3 host=host3" }; - code = taos_schemaless_insert(taos, lines3_1, 3, 1, NULL); + result = taos_schemaless_insert(taos, lines3_1, 3, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("lines3_1 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); return; } @@ -214,10 +236,12 @@ void verify_json_insert(TAOS* taos) { } \ }"}; - code = taos_schemaless_insert(taos, message, 0, 2, NULL); + result = taos_schemaless_insert(taos, message, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload_0 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); char *message1[] = { "[ \ @@ -245,10 +269,12 @@ void verify_json_insert(TAOS* taos) { } \ ]"}; - code = taos_schemaless_insert(taos, message1, 0, 2, NULL); + result = taos_schemaless_insert(taos, message1, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload_1 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); char *message2[] = { "[ \ @@ -296,10 +322,12 @@ void verify_json_insert(TAOS* taos) { } \ } \ ]"}; - code = taos_schemaless_insert(taos, message2, 0, 2, NULL); + result = taos_schemaless_insert(taos, message2, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload_2 code: %d, %s.\n", code, tstrerror(code)); } + taos_free_result(result); cJSON *payload, *tags; @@ -320,12 +348,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload0_0 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //true payload = cJSON_CreateObject(); @@ -341,12 +371,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload0_1 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //false payload = cJSON_CreateObject(); @@ -362,12 +394,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload0_2 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //string payload = cJSON_CreateObject(); @@ -383,12 +417,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload0_3 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //timestamp 0 -> current time payload = cJSON_CreateObject(); @@ -404,12 +440,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload0_4 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); /* Nested format */ //timestamp @@ -433,12 +471,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload1_0 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //milleseconds payload = cJSON_CreateObject(); @@ -459,12 +499,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload1_1 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //microseconds payload = cJSON_CreateObject(); @@ -485,12 +527,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload1_2 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //now payload = cJSON_CreateObject(); @@ -511,12 +555,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload1_4 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //metric value cJSON *metric_val; @@ -543,12 +589,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_0 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //tinyint payload = cJSON_CreateObject(); @@ -573,12 +621,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_1 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //smallint payload = cJSON_CreateObject(); @@ -603,12 +653,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_2 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //int payload = cJSON_CreateObject(); @@ -633,12 +685,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_3 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //bigint payload = cJSON_CreateObject(); @@ -663,12 +717,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_4 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //float payload = cJSON_CreateObject(); @@ -693,12 +749,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_5 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //double payload = cJSON_CreateObject(); @@ -723,12 +781,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_6 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //binary payload = cJSON_CreateObject(); @@ -753,12 +813,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_7 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //nchar payload = cJSON_CreateObject(); @@ -783,12 +845,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload2_8 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); //tag value cJSON *tag; @@ -863,12 +927,14 @@ void verify_json_insert(TAOS* taos) { *payload_str = cJSON_Print(payload); //printf("%s\n", payload_str); - code = taos_schemaless_insert(taos, payload_str, 0, 2, NULL); + result = taos_schemaless_insert(taos, payload_str, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NOT_CONFIGURED); + code = taos_errno(result); if (code) { printf("payload3_0 code: %d, %s.\n", code, tstrerror(code)); } free(*payload_str); cJSON_Delete(payload); + taos_free_result(result); } int main(int argc, char *argv[]) { diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index ec72827c9697cbb30a5845ff5f2a2f809ada4164..ebc054777940430e9cdb78b55b496dda873f2143 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -228,4 +228,6 @@ run general/db/show_create_table.sim run general/parser/like.sim run general/parser/regex.sim run general/parser/tbname_escape.sim +run general/parser/columnName_escape.sim +run general/parser/tagName_escape.sim run general/parser/interp_blocks.sim diff --git a/tests/script/general/compute/csum2.sim b/tests/script/general/compute/csum2.sim index 506070ae369ccb4c1d2bc28d149c7126079a2b54..48de71df394a6f520ede1a2540ad78e215c57075 100644 --- a/tests/script/general/compute/csum2.sim +++ b/tests/script/general/compute/csum2.sim @@ -78,7 +78,9 @@ print ===> $data11 if $data11 != 1.000000000 then return -1 endi + sql_error select csum(c7) from $tb +sql_error select csum(c7) from $tb group by c8 sql_error select csum(c8) from $tb sql_error select csum(c9) from $tb sql_error select csum(ts) from $tb diff --git a/tests/script/general/compute/mavg2.sim b/tests/script/general/compute/mavg2.sim index 60b170e270505b7c3e8d2ee174a4e3b8a4ad223d..55c1fbb29fa276e0c58eb592d910dce8f01da558 100644 --- a/tests/script/general/compute/mavg2.sim +++ b/tests/script/general/compute/mavg2.sim @@ -80,6 +80,7 @@ if $data11 != 1.500000000 then endi sql_error select mavg(c7,2) from $tb sql_error select mavg(c8,2) from $tb +sql_error select mavg(c8,2) from $tb order by c7 sql_error select mavg(c9,2) from $tb sql_error select mavg(ts,2) from $tb sql_error select mavg(c1,2), mavg(c2,2) from $tb diff --git a/tests/script/general/parser/columnName_escape.sim b/tests/script/general/parser/columnName_escape.sim new file mode 100644 index 0000000000000000000000000000000000000000..dd3278d0dc98fa5378b7aed122dc39f6717372d5 --- /dev/null +++ b/tests/script/general/parser/columnName_escape.sim @@ -0,0 +1,426 @@ +system sh/stop_dnodes.sh + + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +sql create database colesc; + +sql use colesc; + +print ======================= test create table/stable + + +sql create table tb0 (ts timestamp, `123` int, `123 456` int, `123.abc` int) +sql create table tb1 (ts timestamp, `!%^&*()` int) +sql create table tb2 (ts timestamp, `int` int, `bool` int, `double` int, `INTO` int, `COLUMN` int) + +sql create table stb0 (ts timestamp, `123` int, `123 456` int, `123.abc` int) tags (t1 int) +sql create table stb1 (ts timestamp, `!%^&*()` int) tags (t2 int) +sql create table stb2 (ts timestamp, `int` int, `bool` int, `double` int, `INTO` int, `COLUMN` int) tags (t3 int) + +sql create table ctb0 using stb0 tags (1) +sql create table ctb1 using stb1 tags (1) +sql create table ctb2 using stb2 tags (1) + +##check table +sql describe tb0; +if $rows != 4 then + return -1 +endi +if $data10 != @123@ then + return -1 +endi +if $data20 != @123 456@ then + return -1 +endi +if $data30 != @123.abc@ then + return -1 +endi + +sql describe tb1; +if $rows != 2 then + return -1 +endi +if $data10 != @!%^&*()@ then + return -1 +endi + +sql describe tb2; +if $rows != 6 then + return -1 +endi +if $data10 != @int@ then + return -1 +endi +if $data20 != @bool@ then + return -1 +endi +if $data30 != @double@ then + return -1 +endi +if $data40 != @INTO@ then + return -1 +endi +if $data50 != @COLUMN@ then + return -1 +endi +##check stable +sql describe stb0; +if $rows != 5 then + return -1 +endi +if $data10 != @123@ then + return -1 +endi +if $data20 != @123 456@ then + return -1 +endi +if $data30 != @123.abc@ then + return -1 +endi + +sql describe stb1; +if $rows != 3 then + return -1 +endi +if $data10 != @!%^&*()@ then + return -1 +endi + +sql describe stb2; +if $rows != 7 then + return -1 +endi +if $data10 != @int@ then + return -1 +endi +if $data20 != @bool@ then + return -1 +endi +if $data30 != @double@ then + return -1 +endi +if $data40 != @INTO@ then + return -1 +endi +if $data50 != @COLUMN@ then + return -1 +endi + + +print ======================= test Alter columns for table/stable + +##Add column +sql_error alter table tb0 add column `123` int +sql_error alter table tb0 add column `123 456` int +sql_error alter table tb0 add column `123.abc` int + +sql_error alter table ctb0 add column `1234` + +sql alter table tb0 add column `!%^&*()` int +sql alter table tb0 add column `int` int +sql alter table tb0 add column `bool` int +sql alter table tb0 add column `double` int +sql alter table tb0 add column `INTO` nchar(10) +sql alter table tb0 add column `COLUMN` binary(10) + +sql alter table stb0 add column `!%^&*()` int +sql alter table stb0 add column `int` int +sql alter table stb0 add column `bool` int +sql alter table stb0 add column `double` int +sql alter table stb0 add column `INTO` nchar(10) +sql alter table stb0 add column `COLUMN` binary(10) + + +##check table +sql describe tb0; +if $rows != 10 then + return -1 +endi +if $data40 != @!%^&*()@ then + return -1 +endi +if $data50 != @int@ then + return -1 +endi +if $data60 != @bool@ then + return -1 +endi +if $data70 != @double@ then + return -1 +endi +if $data80 != @INTO@ then + return -1 +endi +if $data90 != @COLUMN@ then + return -1 +endi + +#check stable +sql describe stb0; +if $rows != 11 then + return -1 +endi +if $data40 != @!%^&*()@ then + return -1 +endi +if $data50 != @int@ then + return -1 +endi +if $data60 != @bool@ then + return -1 +endi +if $data70 != @double@ then + return -1 +endi +if $data80 != @INTO@ then + return -1 +endi +if $data90 != @COLUMN@ then + return -1 +endi + +##Drop column + +sql_error alter table ctb0 drop column `123` +sql_error alter table ctb0 drop column `123 456` +sql_error alter table ctb0 drop column `123.abc` + +sql alter table tb0 drop column `!%^&*()` +sql alter table tb0 drop column `int` +sql alter table tb0 drop column `bool` +sql alter table tb0 drop column `double` +sql alter table tb0 drop column `INTO` +sql alter table tb0 drop column `COLUMN` + +sql alter table stb0 drop column `!%^&*()` +sql alter table stb0 drop column `int` +sql alter table stb0 drop column `bool` +sql alter table stb0 drop column `double` +sql alter table stb0 drop column `INTO` +sql alter table stb0 drop column `COLUMN` + +##check table +sql describe tb0; +if $rows != 4 then + return -1 +endi +if $data10 != @123@ then + return -1 +endi +if $data20 != @123 456@ then + return -1 +endi +if $data30 != @123.abc@ then + return -1 +endi + +##check stable +sql describe stb0; +if $rows != 5 then + return -1 +endi +if $data10 != @123@ then + return -1 +endi +if $data20 != @123 456@ then + return -1 +endi +if $data30 != @123.abc@ then + return -1 +endi + +##Modify column for binary/nchar length + +sql alter table tb0 add column `INTO` nchar(10) +sql alter table tb0 add column `COLUMN` binary(10) + +sql alter table stb0 add column `INTO` nchar(10) +sql alter table stb0 add column `COLUMN` binary(10) + +sql alter table tb0 modify column `INTO` nchar(15) +sql alter table tb0 modify column `COLUMN` binary(15) + +sql alter table stb0 modify column `INTO` nchar(15) +sql alter table stb0 modify column `COLUMN` binary(15) + +sql describe tb0; +if $rows != 6 then + return -1 +endi +if $data42 != @15@ then + return -1 +endi +if $data52 != @15@ then + return -1 +endi + +sql describe stb0; +if $rows != 7 then + return -1 +endi +if $data42 != @15@ then + return -1 +endi +if $data52 != @15@ then + return -1 +endi + +print ======================= test insert columns for table/stable + +sql insert into tb0 (ts, `123`, `123 456`, `123.abc`) values (now, 1, 1, 1) +sql insert into tb1 (ts, `!%^&*()`) values (now, 1) +sql insert into tb2 (ts, `int`, `bool`, `double`, `INTO`, `COLUMN`) values (now, 1, 1, 1, 1, 1) + +sql insert into ctb0 (ts, `123`, `123 456`, `123.abc`) values (now, 1, 1, 1) +sql insert into ctb1 (ts, `!%^&*()`) values (now, 1) +sql insert into ctb2 (ts, `int`, `bool`, `double`, `INTO`, `COLUMN`) values (now, 1, 1, 1, 1, 1) + +sql select * from tb0; +if $rows != 1 then + return -1 +endi + +sql select * from tb1; +if $rows != 1 then + return -1 +endi + +sql select * from tb2; +if $rows != 1 then + return -1 +endi + +sql select * from ctb0; +if $rows != 1 then + return -1 +endi + +sql select * from ctb1; +if $rows != 1 then + return -1 +endi + +sql select * from ctb2; +if $rows != 1 then + return -1 +endi + +print ======================= test select columns for table/stable + +sql select `123`,`123 456`,`123.abc` from tb0; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select `!%^&*()` from tb1; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select `int`,`bool`,`double`,`INTO`,`COLUMN` from tb2; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + + +sql select `123`,`123 456`,`123.abc` from stb0; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select `!%^&*()` from stb1; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select `int`,`bool`,`double`,`INTO`,`COLUMN` from stb2; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/tagName_escape.sim b/tests/script/general/parser/tagName_escape.sim new file mode 100644 index 0000000000000000000000000000000000000000..9b04dc2899af0dc4489f8f5621bbd2db3a79789b --- /dev/null +++ b/tests/script/general/parser/tagName_escape.sim @@ -0,0 +1,186 @@ +system sh/stop_dnodes.sh + + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +sql create database tagesc; + +sql use tagesc; + +print ======================= test create table/stable + +sql create stable stb0 (ts timestamp, c0 int) tags (`123` int, `123 456` int, `123.abc` int) +sql create stable stb1 (ts timestamp, c1 int) tags (`!%^&*()` int) +sql create stable stb2 (ts timestamp, c2 int) tags (`int` int, `bool` int, `double` int, `INTO` int, `COLUMN` int) + +sql create table ctb0 using stb0 (`123`, `123 456`, `123.abc`) tags (1, 1, 1) +sql create table ctb1 using stb1 (`!%^&*()`) tags (1) +sql create table ctb2 using stb2 (`int`, `bool`, `double`, `INTO`, `COLUMN`) tags (1, 1, 1, 1, 1) + +##check table +sql describe ctb0; +if $rows != 5 then + return -1 +endi +if $data20 != @123@ then + return -1 +endi +if $data30 != @123 456@ then + return -1 +endi +if $data40 != @123.abc@ then + return -1 +endi + +sql describe ctb1; +if $rows != 3 then + return -1 +endi +if $data20 != @!%^&*()@ then + return -1 +endi + +sql describe ctb2; +if $rows != 7 then + return -1 +endi +if $data20 != @int@ then + return -1 +endi +if $data30 != @bool@ then + return -1 +endi +if $data40 != @double@ then + return -1 +endi +if $data50 != @INTO@ then + return -1 +endi +if $data60 != @COLUMN@ then + return -1 +endi + +print ======================= test Alter tags for stable + +##ADD TAG +sql_error alter stable stb0 add tag `123` int +sql_error alter stable stb0 add tag `123 456` int +sql_error alter stable stb0 add tag `123.abc` int + +sql alter stable stb0 add tag `!%^&*()` int +sql alter stable stb0 add tag `int` int +sql alter stable stb0 add tag `bool` int +sql alter stable stb0 add tag `double` int +sql alter stable stb0 add tag `INTO` int +sql alter stable stb0 add tag `COLUMN` int + + +sql describe stb0; +if $rows != 11 then + return -1 +endi +if $data50 != @!%^&*()@ then + return -1 +endi +if $data60 != @int@ then + return -1 +endi +if $data70 != @bool@ then + return -1 +endi +if $data80 != @double@ then + return -1 +endi +if $data90 != @INTO@ then + return -1 +endi + + +##DROP TAG +sql alter stable stb0 drop tag `!%^&*()` +sql alter stable stb0 drop tag `int` +sql alter stable stb0 drop tag `bool` +sql alter stable stb0 drop tag `double` +sql alter stable stb0 drop tag `INTO` +sql alter stable stb0 drop tag `COLUMN` + + +sql describe stb0; +if $rows != 5 then + return -1 +endi +if $data20 != @123@ then + return -1 +endi +if $data30 != @123 456@ then + return -1 +endi +if $data40 != @123.abc@ then + return -1 +endi + + +##CHANGE TAG + +sql alter stable stb0 change tag `123` `321` +sql alter stable stb0 change tag `123 456` `456 123` +#sql alter stable stb0 change tag `123.abc` `abc.123` +#change tag has bug when using dot in tagname + +sql describe stb0; +if $rows != 5 then + return -1 +endi +if $data20 != @321@ then + return -1 +endi +if $data30 != @456 123@ then + return -1 +endi + +##SET TAG + +sql insert into ctb0 values (now, 1) +sql insert into ctb1 values (now, 1) +sql insert into ctb2 values (now, 1) + +sql alter table ctb0 set tag `321`=2 +sql alter table ctb0 set tag `456 123`=2 +#sql alter table ctb0 set tag `abc.123`=2 +#change tag has bug when using dot in tagname + +print ======================= test insert specific tags automatically create table + +sql alter table ctb0 set tag `321`=2 +sql alter table ctb0 set tag `321`=2 +sql insert into ctb0_0 using stb0 (`321`, `456 123`, `123.abc`) tags (1, 1, 1) values (now + 10s, 5) +sql insert into ctb1_0 using stb1 (`!%^&*()`) tags (1) values (now + 10s, 5) +sql insert into ctb2_0 using stb2 (`int`, `bool`, `double`, `INTO`, `COLUMN`) tags (1, 1, 1, 1, 1) values (now + 10s, 5) +sql insert into ctb2_1 using stb2 (`int`, `bool`, `INTO`, `COLUMN`) tags (1, 1, 1, 1) values (now + 10s, 5) + +sql select * from stb0; +if $rows != 2 then + return -1 +endi + +sql select * from stb1; +if $rows != 2 then + return -1 +endi + +sql select * from stb2; +if $rows != 3 then + return -1 +endi + +if $data24 != NULL then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/udf_dll.sim b/tests/script/general/parser/udf_dll.sim index 7168e0a5ddf5502170e6bb22f30b10621795a568..61bf5fee6e54d02ccc08218102a43a37821fdd30 100644 --- a/tests/script/general/parser/udf_dll.sim +++ b/tests/script/general/parser/udf_dll.sim @@ -452,6 +452,7 @@ if $data31 != 2 then return -1 endi +sql_error select add_one(f1) from tb1 order by ts desc; sql select add_one(f1) from tb1 limit 2; if $rows != 2 then diff --git a/tests/script/general/parser/udf_dll_stable.sim b/tests/script/general/parser/udf_dll_stable.sim index 15becaab22476d12829abc62db4de4f914eef271..cd1dbc8b5374779d13decde5bf8a0fce48d90f0a 100644 --- a/tests/script/general/parser/udf_dll_stable.sim +++ b/tests/script/general/parser/udf_dll_stable.sim @@ -10,9 +10,10 @@ sql connect print ======================== dnode1 start sql create function add_one as '/tmp/add_one.so' outputtype int; +sql create function add_one_64232 as '/tmp/add_one_64232.so' outputtype int; sql create aggregate function sum_double as '/tmp/sum_double.so' outputtype int; sql show functions; -if $rows != 2 then +if $rows != 3 then return -1 endi @@ -1154,6 +1155,93 @@ if $data61 != 22 then return -1 endi +sql_error select sum_double(f1),add_one(f1) from tb1 where ts>="2021-03-23 17:00:00.000" and ts<="2021-03-24 20:00:00.000" interval (1h) sliding (30m); + +sql select add_one(f1) from (select * from tb1); +if $rows != 7 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi +if $data10 != 3 then + return -1 +endi +if $data20 != 4 then + return -1 +endi +if $data30 != 5 then + return -1 +endi +if $data40 != 6 then + return -1 +endi +if $data50 != 7 then + return -1 +endi +if $data60 != 8 then + return -1 +endi + +sql select add_one(ff1) from (select add_one(f1) as ff1 from tb1); +if $rows != 7 then + return -1 +endi + +if $data00 != 3 then + return -1 +endi +if $data10 != 4 then + return -1 +endi +if $data20 != 5 then + return -1 +endi +if $data30 != 6 then + return -1 +endi +if $data40 != 7 then + return -1 +endi +if $data50 != 8 then + return -1 +endi +if $data60 != 9 then + return -1 +endi + +sql_error select add_one(f1),sub_one(f1) from tb1; + + +sql create table taaa (ts timestamp, f1 bigint); +sql insert into taaa values (now, 1); +sleep 100 +sql insert into taaa values (now, 10); +sleep 100 +sql insert into taaa values (now, 1000); +sleep 100 +sql insert into taaa values (now, 100); + +sql select add_one_64232(f1) from taaa; +if $rows != 4 then + print $rows + return -1 +endi + +if $data00 != 2 then + return -1 +endi +if $data10 != 11 then + return -1 +endi +if $data20 != 1001 then + return -1 +endi +if $data30 != 101 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a9b2764495095b86c55f56c52c55c74f4e545e96..850f3a19467a8748bba56f80033d4fc0b0bc77a3 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -404,7 +404,7 @@ cd ../../../debug; make ./test.sh -f unique/mnode/mgmt34.sim ./test.sh -f unique/mnode/mgmtr2.sim -./test.sh -f unique/arbitrator/insert_duplicationTs.sim +#./test.sh -f unique/arbitrator/insert_duplicationTs.sim ./test.sh -f general/parser/join_manyblocks.sim ./test.sh -f general/parser/stableOp.sim ./test.sh -f general/parser/timestamp.sim @@ -415,4 +415,8 @@ cd ../../../debug; make ./test.sh -f general/parser/last_cache.sim ./test.sh -f unique/big/balance.sim +./test.sh -f general/parser/udf.sim +./test.sh -f general/parser/udf_dll.sim +./test.sh -f general/parser/udf_dll_stable.sim + #======================b7-end=============== diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c index 2983ad1a43d60494f75d32978ca51c5e385fa0b2..e9f11feb414363eb0e741c722f4d4dd79b87e81e 100644 --- a/tests/script/sh/abs_max.c +++ b/tests/script/sh/abs_max.c @@ -38,6 +38,8 @@ void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts *(long *)dataOutput=r; printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); + } else { + *numOfOutput=0; } } @@ -47,7 +49,7 @@ void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfIn int i; int r = 0; printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf); - *numOfOutput=1; + printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); } diff --git a/tests/script/sh/add_one_64232.c b/tests/script/sh/add_one_64232.c new file mode 100644 index 0000000000000000000000000000000000000000..8db87d049d607a7fed60580dcdaf682dcca944b4 --- /dev/null +++ b/tests/script/sh/add_one_64232.c @@ -0,0 +1,33 @@ +#include +#include +#include + +typedef struct SUdfInit{ + int maybe_null; /* 1 if function can return NULL */ + int decimals; /* for real functions */ + long long length; /* For string functions */ + char *ptr; /* free pointer for function data */ + int const_item; /* 0 if result is independent of arguments */ +} SUdfInit; + +void add_one_64232(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput, + int* numOfOutput, short otype, short obytes, SUdfInit* buf) { + int i; + int r = 0; + printf("add_one_64232 input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); + if (itype == 5) { + for(i=0;i +#include +#include + +typedef struct SUdfInit{ + int maybe_null; /* 1 if function can return NULL */ + int decimals; /* for real functions */ + long long length; /* For string functions */ + char *ptr; /* free pointer for function data */ + int const_item; /* 0 if result is independent of arguments */ +} SUdfInit; + +void sub_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput, + int* numOfOutput, short otype, short obytes, SUdfInit* buf) { + int i; + int r = 0; + printf("sub_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); + if (itype == 4) { + for(i=0;ifileName, rest); simLogSql(buf, true); - char * lines[] = {rest}; - int32_t ret = taos_schemaless_insert(script->taos, lines, 1, 0, "ns"); - if (ret == TSDB_CODE_SUCCESS) { + char* lines[] = {rest}; + TAOS_RES *result = taos_schemaless_insert(script->taos, lines, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + int32_t code = taos_errno(result); + if (code == TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); script->linePos++; - return true; + ret = true; } else { - sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, - ret & 0XFFFF, tstrerror(ret)); - return false; + sprintf(script->error, "lineNum: %d. line: %s failed, code:%d:%s", line->lineNum, rest, + code & 0XFFFF, taos_errstr(result)); + ret = false; } + taos_free_result(result); + return ret; } bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { + bool ret; char buf[TSDB_MAX_BINARY_LEN]; simVisuallizeOption(script, rest, buf); @@ -1107,14 +1112,17 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); char * lines[] = {rest}; - int32_t ret = taos_schemaless_insert(script->taos, lines, 1, 0, "ns"); - if (ret == TSDB_CODE_SUCCESS) { + TAOS_RES *result = taos_schemaless_insert(script->taos, lines, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + int32_t code = taos_errno(result); + if (code == TSDB_CODE_SUCCESS) { sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest); script->linePos++; - return false; + ret = false; } else { - simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, - ret & 0XFFFF, tstrerror(ret)); - return true; + simDebug("lineNum: %d. line: %s failed, code:%d:%s. Expect failed, so success", line->lineNum, rest, + code & 0XFFFF, taos_errstr(result)); + ret = true; } + taos_free_result(result); + return ret; }